python 爬取中国房价行情网网站

远方的飞猪

已于 2023-07-30 14:46:55 修改

阅读量841

点赞数 1

分类专栏：爬虫文章标签： python

于 2020-09-19 00:37:13 首次发布

本文链接：https://blog.csdn.net/tanjunchen/article/details/108675654

版权

本文介绍了如何使用Python编写爬虫程序，从中国房价行情网抓取房地产数据。通过提供的GitHub链接，可以获取名为HousePriceSpider.py的完整代码示例，帮助读者了解和学习网络爬虫的实现。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

HousePriceSpider.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
import json
import time
from multiprocessing import Pool, Manager, cpu_count
from urllib.parse import urljoin
import numpy as np
import pandas as pd
import hashlib
from lxml import etree
from ChinaHousePrice.session import SessionWrapper
from datetime import datetime
from ChinaHousePrice.common import AreaCodeDecoder
import pymysql
from sqlalchemy import create_engine
from ChinaHousePrice import config
from sqlalchemy.sql import text


class Spider(object):
    def __init__(self, prov, city, url):
        self.prov = prov
        self.city = city
        self.url = url
        self.__session = SessionWrapper(timeout=20)
        self.__href_d = self._district()
        self.city_data = self._city_data()

    def _district(self):
        href_d = {}
        res = self.__session.get(self.url)
        if res is None:
            return href_d
        html = etree.HTML(res.text)
        href_n = html.xpath("//span[@class='city-n']/a/@href")
        district_name_n = html.xpath("//span[@class='city-n']/a/span/text()")
        href_d.update(dict(zip(district_name_n, href_n)))
        href_w = html.xpath("//span[@class='city-w']/a/@href")
        district_name_w = html.xpath("//span[@class='city-w']/a/span/text()")
        href_d.update(dict(zip(district_name_w, href_w)))
        return href_d

    def _city_data(self):
        return self._district_all_data(np.nan, self.url + "?")

    def _district_all_data(self, district, url):
        params = ["", "?&type=newha", "?&type=lease", "?&type=lease&proptype=22", "?&proptype=22", "?&proptype=21",
                  "?&type=lease&proptype=22"]
        result = [self.prov, self.city, district]
        for u in params:
            url_ = url + u
            result += self._parse_by_xpath(url_)
            print("Crawl-->" + url_, self._parse_by_xpath(url_))
        # prov,city,district,住宅+二手房