python二手房数据分析_使用python3抓取链家二手房数据

import requests

from bs4 import BeautifulSoup

import sys

import os

import time

import pandas as pd

import numpy as np

from parsel import Selector

import re

headers = {

'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.106 BIDUBrowser/8.7 Safari/537.36'

}

def catchHouseList(url):

resp = requests.get(url, headers=headers, stream=True)

if resp.status_code == 200:

reg = re.compile('

.*?

urls = re.findall(reg, resp.text)

return urls

return []

def catchHouseDetail(url):

resp = requests.get(url, headers=headers)

print(url)

if resp.status_code == 200:

info = {}

soup = BeautifulSoup(resp.text, 'html.parser')

info['标题'] = soup.select('.main')[0].text

info['总价'] = soup.select('.total')[0].text

info['总价单位'] = soup.select('.unit')[0].text

info['每平方售价'] = soup.select('.unitPriceValue')[0].text

# p = soup.select('.tax')

# info['参考总价'] = soup.select('.tax')[0].text

info['建造时间'] = soup.select('.subInfo')[2].text

info['小区名称'] = soup.select('.info')[0].text

info['所在区域'] = soup.select('.info a')[0].text + ':' + soup.select('.info a')[1].text

info['链家编号'] = str(url)[34:].rsplit('.html')[0]

info['房屋户型'] = str(soup.select('.content')[2].select('.label')[0].next_sibling)

info['所在楼层'] = soup.select('.content')[2].select('.label')[1].next_sibling

info['建筑面积'] = soup.select('.content')[2].select('.label')[2].next_sibling

info['户型结构'] = soup.select('.content')[2].select('.label')[3].next_sibling

info['套内面积'] = soup.select('.content')[2].select('.label')[4].next_sibling

info['建筑类型'] = soup.select('.content')[2].select('.label')[5].next_sibling

info['房屋朝向'] = soup.select('.content')[2].select('.label')[6].next_sibling

info['建筑结构'] = soup.select('.content')[2].select('.label')[7].next_sibling

info['装修情况'] = soup.select('.content')[2].select('.label')[8].next_sibling

info['梯户比例'] = soup.select('.content')[2].select('.label')[9].next_sibling

info['供暖方式'] = soup.select('.content')[2].select('.label')[10].next_sibling

info['配备电梯'] = soup.select('.content')[2].select('.label')[11].next_sibling

#  info['产权年限'] = str(soup.select('.content')[2].select('.label')[12].next_sibling)

return info

pass

def appendToXlsx(info):

fileName = './链家二手房.xlsx'

dfNew = pd.DataFrame([info])

if(os.path.exists(fileName)):

sheet = pd.read_excel(fileName)

dfOld = pd.DataFrame(sheet)

df = pd.concat([dfOld, dfNew])

df.to_excel(fileName)

else:

dfNew.to_excel(fileName)

def catch():

pages = ['https://sz.lianjia.com/ershoufang/pg{}/'.format(x) for x in range(1, 1001)]

for page in pages:

print(page)

houseListURLs = catchHouseList(page)

for houseDetailUrl in houseListURLs:

try:

info = catchHouseDetail(houseDetailUrl)

appendToXlsx(info)

except:

pass

time.sleep(3)

pass

if __name__ == '__main__':

catch()