爬取链家二手房数据

1.导入工具

import requests
import parsel
import csv
from lxml import html 

2.请求网页响应，爬取数据，并将爬取到的文件写入文档

etree = html.etree      #解析字符串格式的HTML文档对象
headers ={
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.107 Safari/537.36 Edg/92.0.902.55'
}


with open('5885_all_1.csv','w',encoding='utf-8', newline='') as my_csv:

    csv_writer = csv.writer(my_csv)

3.开始遍历想要爬取的页数

for page in range(1,30):
        url ='https://bj.lianjia.com/ershoufang/pg' + str(page) + '/'
        response = requests.get(url=url,headers=headers)          #向网站发起请求，并获取响应对象
        #print(page_text)
        tree = etree.HTML(page_text)

        selector = parsel.Selector(response.text)
        lis = selector.css('.sellListContent li')

4.开始提取网页数据

for li in lis:
            #print(1111111111)
            dit = {}
            title = li.css('.title a::text').get()
            dit['标题'] = title
            positionInfo = li.css('.positionInfo a::text').getall()
            info = '-'.join(positionInfo)
            dit['开发商'] = info
            houseInfo = li.css('.houseInfo::text').get()
            dit['房子信息'] = houseInfo
            followInfo = li.css('.followInfo::text').get()
            dit['发布周期'] = followInfo
            Price = li.css('.totalPrice span::text').get()
            dit['售价/万'] = Price
            unitPrice = li.css('.unitPrice span::text').get()
            dit['单价'] = unitPrice
            print(dit)

5.开始输出数据

csv_writer.writerow([dit['标题'],dit['开发商'],dit['房子信息'], dit['发布周期'], dit['售价/万'], dit['单价']])
        print('page',page,'ok')

6.数据可视化部分结果演示