Python爬取链家房产信息

Python爬取链家房产信息

需要的库

• requests
• BeautifulSoup
• re
• pandas

程序如下：

import requests
from bs4 import BeautifulSoup
import pandas as pd
import re
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from matplotlib.font_manager import FontProperties
myfont=FontProperties(fname=r'C:\Windows\Fonts\simhei.ttf',size=1)
sns.set(font=myfont.get_name())


def removenone(mylist):

    while '' in mylist:
        mylist.remove('')
    
    return mylist

def addnone(mylist,length,cha):
    while len(mylist) < length:
        mylist.append(cha)
    return mylist

def regnum(s):
    mylist = re.findall(r'[\d+\.\d]*', s)
    mylist = removenone(mylist)
    return mylist

def lianjia(url,page_range,district):
    #Initialization
    colum_name = ['Title','Position','Tag','followInfo','VR','Info','Total price','RMB/m^2','Attention','Update day','Bed room','Living room','Area','Floors','Year','WebPage']
    data_list = []
    for page in range(page_range):
        pgurl = url+'/pg'+str(page+1)
        print (pgurl)
        header = {
            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.109 Safari/537.36'}
        page = requests.get(pgurl, headers=header)  #访问网址 获取该 html内容
        a = page.text
        
        soup = BeautifulSoup(a,"lxml")  #解析该网页内容
        for b in soup.find_all('div',class_='info clear'): #find_all 找到 div  class='info clear' 的标签
            temp = []
            for wz in b.find_all('div',class_ = ['title','positionInfo','tag','houseInfo','priceInfo','followInfo']):
                temp.append(wz.get_text())
            # price = temp[3].split('万')
            # price = temp[3].split('元')
            tag = regnum(temp[2])
            date = regnum(temp[3])
            date = addnone(date,2,'>365')
            price = regnum(temp[5])
            temp.extend(price)
            temp.extend(date)

            temp.extend(tag)
            # print (temp)
            temp = addnone(temp,15,'0') 
            for title in b.find_all('div',class_ = 'title'):
                for link in title.find_all('a'):  
                    temp.append(link.get('href'))        
            # print (temp)
            # for pos in b.find_all('div',class_ = 'positionInfo'): 
            #     temp.append(pos.get_text())
            data_list.append(temp)	    
    data = pd.DataFrame(data_list,columns=colum_name)
    data[['Total price','RMB/m^2','Area']]=data[['Total price','RMB/m^2','Area']].astype('float')
    data.to_csv(district+'.csv',encoding='utf_8_sig')
    return data

if __name__ == '__main__':
    #User defined
    district_list = ['putuo']
    for district in district_list:
        url = "https://sh.lianjia.com/ershoufang/"+district
        # url = "https://xinxiang.lianjia.com/ershoufang/"+district
        # url = "https://sh.lianjia.com/weifang"
        page_range=100
        my = lianjia(url,page_range,district)
        DG = my[my['Update day']!='>365'][['Update day','Position','Total price','RMB/m^2','Area']].groupby("Position")
        res = DG.agg([np.mean,np.std])
        name = [i.split()[0] for i in res.index] 
        plt.scatter(name,res['RMB/m^2']['mean'])
        plt.xticks(rotation=45,size =4)
        plt.savefig("xuhui.png")