python 爬取中国房价行情网网站

HousePriceSpider.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
import json
import time
from multiprocessing import Pool, Manager, cpu_count
from urllib.parse import urljoin
import numpy as np
import pandas as pd
import hashlib
from lxml import etree
from ChinaHousePrice.session import SessionWrapper
from datetime import datetime
from ChinaHousePrice.common import AreaCodeDecoder
import pymysql
from sqlalchemy import create_engine
from ChinaHousePrice import config
from sqlalchemy.sql import text


class Spider(object):
    def __init__(self, prov, city, url):
        self.prov = prov
        self.city = city
        self.url = url
        self.__session = SessionWrapper(timeout=20)
        self.__href_d = self._district()
        self.city_data = self._city_data()

    def _district(self):
        href_d = {}
        res = self.__session.get(self.url)
        if res is None:
            return href_d
        html = etree.HTML(res.text)
        href_n = html.xpath("//span[@class='city-n']/a/@href")
        district_name_n = html.xpath("//span[@class='city-n']/a/span/text()")
        href_d.update(dict(zip(district_name_n, href_n)))
        href_w = html.xpath("//span[@class='city-w']/a/@href")
        district_name_w = html.xpath("//span[@class='city-w']/a/span/text()")
        href_d.update(dict(zip(district_name_w, href_w)))
        return href_d

    def _city_data(self):
        return self._district_all_data(np.nan, self.url + "?")

    def _district_all_data(self, district, url):
        params = ["", "?&type=newha", "?&type=lease", "?&type=lease&proptype=22", "?&proptype=22", "?&proptype=21",
                  "?&type=lease&proptype=22"]
        result = [self.prov, self.city, district]
        for u in params:
            url_ = url + u
            result += self._parse_by_xpath(url_)
            print("Crawl-->" + url_, self._parse_by_xpath(url_))
        # prov,city,district,住宅+二手房