python二手房使用教程_python链家二手房爬虫实战

import requests

import time

from bs4 import BeautifulSoup

import pandas as pd

import numpy as np

import seaborn as sns

import matplotlib as mpl

import matplotlib.pyplot as plt

from IPython.display import display

plt.style.use("fivethirtyeight")

sns.set_style({'font.sans-serif':['simhei','Arial']})

%matplotlib inline

from matplotlib.font_manager import FontProperties

font_zh = FontProperties(fname='/Library/Fonts/SimHei.ttc')

#设置列表页URL的固定部分

url='http://hz.lianjia.com/ershoufang/'

#设置页面页的可变部分

page=('pg')

#设置请求头部信息

headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',

'Accept':'text/html;q=0.9,*/*;q=0.8',

'Accept-Charset':'ISO-8859-1,utf-8;q=0.7,*;q=0.3',

'Accept-Encoding':'gzip',

'Connection':'close',

'Referer':'http://www.baidu.com/link?url=_andhfsjjjKRgEWkj7i9cFmYYGsisrnm2A-TN3XZDQXxvGsM9k9ZZSnikW2Yds4s&wd=&eqid=c3435a7d00006bd600000003582bfd1f'

}

#循环抓取列表页信息

for i in range(1,100):

if i == 1:

i=str(i)

a=(url+page+i+'/')

r=requests.get(url=a,headers=headers)

html=r.content

else:

i=str(i)

a=(url+page+i+'/')

r=requests.get(url=a,headers=headers)

html2=r.content

html = html + html2

#每次间隔0.5秒

time.sleep(0.5)

print(html)

lj=BeautifulSoup(html,'html.parser')

提取相关数据，之前在网上看过教程，但到实际爬取的时候，老是报错，原来是链家改版了，相应的代码也需要改

#提取房源总价

price=lj.find_all('div',attrs={'class':'priceInfo'})

tp=[]

for a in price:

totalPrice=a.span.string

tp.append(totalPrice)

#提取房源信息

houseInfo=lj.find_all('div',{'class':'houseInfo'})

hi=[]

for b in houseInfo:

house=b.get_text()

hi.append(house)

#提取房源关注度

followInfo=lj.find_all('div',attrs={'class':'followInfo'})

fi=[]

for c in followInfo:

follow=c.get_text()

fi.append(follow)

#提取位置信息

positionInfo = lj.find_all('div',attrs = {'class':'positionInfo'})

pi=[]

for p in positionInfo:

position = p.get_text()

pi.append(position)

提取后进行数据处理

#导入pandas库

import pandas as pd

#创建数据表

house=pd.DataFrame({'totalprice':tp,'houseinfo':hi,'followinfo':fi,'positioninfo':pi})

#查看数据表的内容

house.head()

house['xiaoqu']=house['houseinfo'].apply(lambda x : x.split('|')[0].strip())

house['huxing']=house['houseinfo'].apply(lambda x : x.split('|')[1].strip())

house['mianji']=house['houseinfo'].apply(lambda x : x.split('|')[2].strip())

house['chaoxiang']=house['houseinfo'].apply(lambda x : x.split('|')[3].strip())

house['zhuangxiu']=house['houseinfo'].apply(lambda x : x.split('|')[4].strip())

#house['dianti']=house['houseinfo'].apply(lambda x : x.split('|')[5].strip())

house['guanzhu']=house['followinfo'].apply(lambda x : x.split('/')[0].strip())

house['shijian']=house['followinfo'].apply(lambda x : x.split('/')[2].strip())

house['lougao']=house['positioninfo'].apply(lambda x:x.split('-')[0].strip())

house['weizhi']=house['positioninfo'].apply(lambda x:x.split('-')[1].strip())

import re

def get_num(string):

return(re.findall("\d+\.?\d*",string)[0])

house['mianji_num']=house.mianji.apply(get_num)

house['mianji_num']=house['mianji_num'].astype(float)

house.head()

house['guanzhu_num']=house.guanzhu.apply(get_num)

house.head()

house[['guanzhu_num','totalprice']]=house[['guanzhu_num','totalprice']].astype(float)

house.head()

huxing=house.groupby(['huxing'])['huxing'].agg('count')

house["danjia"]=(house['totalprice']/house['mianji_num'])

house.head()