HousePriceSpider.py
#!/usr/bin/env python # -*- coding: utf-8 -*- import json import time from multiprocessing import Pool, Manager, cpu_count from urllib.parse import urljoin import numpy as np import pandas as pd import hashlib from lxml import etree from ChinaHousePrice.session import SessionWrapper from datetime import datetime from ChinaHousePrice.common import AreaCodeDecoder import pymysql from sqlalchemy import create_engine from ChinaHousePrice import config from sqlalchemy.sql import text class Spider(object): def __init__(self, prov, city, url): self.prov = prov self.city = city self.url = url self.__session = SessionWrapper(timeout=20) self.__href_d = self._district() self.city_data = self._city_data() def _district(self): href_d = {} res = self.__session.get(self.url) if res is None: return href_d html = etree.HTML(res.text) href_n = html.xpath("//span[@class='city-n']/a/@href") district_name_n = html.xpath("//span[@class='city-n']/a/span/text()") href_d.update(dict(zip(district_name_n, href_n))) href_w = html.xpath("//span[@class='city-w']/a/@href") district_name_w = html.xpath("//span[@class='city-w']/a/span/text()") href_d.update(dict(zip(district_name_w, href_w))) return href_d def _city_data(self): return self._district_all_data(np.nan, self.url + "?") def _district_all_data(self, district, url): params = ["", "?&type=newha", "?&type=lease", "?&type=lease&proptype=22", "?&proptype=22", "?&proptype=21", "?&type=lease&proptype=22"] result = [self.prov, self.city, district] for u in params: url_ = url + u result += self._parse_by_xpath(url_) print("Crawl-->" + url_, self._parse_by_xpath(url_)) # prov,city,district,住宅+二手房