北京二手房链家网数据爬取

直接放代码

# -*- coding: utf-8 -*-
# @Time    : 2022/12/23 20:46

import re  # 正则表达式
import json
from numpy import *  # 创建给定类型的矩阵，并初始化为0
import urllib.request,urllib.error

location  = re.compile(r'data-el="region">(.*?)</a>', re.S)
houseInfo = re.compile(r'class="houseIcon"></span>(.*?)</div>', re.S)
years = re.compile(r'')
priceall = re.compile(r'<div class="priceInfo">(.*?)</span>', re.S)
priceaverage = re.compile(r'data-price="(.*?)"',re.S)
four_location_C = ['东城','西城','朝阳','海淀']
four_location_P = ['dongcheng','xicheng','chaoyang','haidian']
def get_data():
    '''
    通过循环获取租房信息
    :return:
    '''
    four_lo = {}
    for j in range(4):
        five_page = {} #爬取5页
        k = 1
        for i in range(1, 6):
            print('正在爬取第%d页' % (i))
            # 爬取成都链家网租房信息
            baseurl = 'https://bj.lianjia.com/ershoufang/'+four_location_P[j]+'/pg'
            url = baseurl + str(i)
            header = {
                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'}
            request = urllib.request.Request(url, headers=header)
            html = ""
            response = urllib.request.urlopen(request)
            html = response.read().decode("utf-8")

            bian1 = re.compile(r'\u2022')
            bian2 = re.compile(r'\xa9')
            con = re.sub(bian1, '', html)
            con = re.sub(bian2, '', con)

            locat = re.findall(location, con)
            house = re.findall(houseInfo, con)
            price1 = re.findall(priceall, con)
            price2 = re.findall(priceaverage, con)

            #one_page = []
            for i in range(len(locat)):
                one = {}
                one['楼盘'] = locat[i]
                one['总价'] = re.findall("[0-9]+", price1[i])[1] + '万'
                one['面积'] = house[i].split('|')[1]
                print(house[i])
                if '年建' in house[i]:
                    t = house[i].index('年建')
                    one['建造时间'] = house[i][t - 4:t]
                else:
                    one['建造时间'] = 0
                one['单价']=price2[i] + '元/平'
                five_page[k]=one
                k = k+1

        four_lo[four_location_C[j]]=five_page
    with open("北京二手房.json", "w", encoding='utf-8') as f:  ## 设置'utf-8'编码
        f.write(json.dumps(four_lo, ensure_ascii=False,indent=4 ))


get_data()