python爬房源信息_Python爬虫：Q房网房源信息

#爬虫项目采取xpath解析

#爬取Q房源网的详情信息并保存为csv文件

#爬取具体内容有："小区名称", "户型", "面积", "装修", "楼层", "朝向",

#  "售价", "总价/万", "详情"

1、导入模块

import requests

import time

from lxml import etree

import csv

2、#定义spider_page()函数爬取并返回页面信息

def spider_page(url):

headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) \

AppleWebKit/537.36 (KHTML, like Gecko) \

Chrome/70.0.3538.110 Safari/537.36',

'upgrade-insecure-requests': '1',

'cookie':'acw_tc=df6fef1a15477176336286817eeb02a7224b5ac26463f80afbe8cf7952; qchatid=d59ef744-850a-427b-9340-264de69f268b; WINDOW_DEVICE_PIXEL_RATIO=1; _ga=GA1.3.1616010054.1547717677; sid=373c20fd-15cf-452e-aaab-d574fa5756c0; _jzqckmp=1; _gid=GA1.3.1123773018.1550143142; cookieId=5068cade-858f-47cd-935a-cb3a511995ac; CITY_NAME=SHENZHEN; sec_tc=AQAAAA9z2QkSZAAAt9QQ9By0mazxhvEk; acw_sc__v2=5c65796b40cb4b7fcb1e52469c34bf5ad61e042a; JSESSIONID=aaaWmDq43YXm_d82KySJw; _qzja=1.1620652135.1547717679057.1550143142187.1550154096058.1550143487410.1550154096058.0.0.0.8.3; _qzjc=1; _qzjto=5.2.0; Hm_lvt_de678bd934b065f76f05705d4e7b662c=1547717676,1550143142,1550154096; Hm_lpvt_de678bd934b065f76f05705d4e7b662c=1550154096; _dc_gtm_UA-47416713-1=1; _jzqa=1.827135515274435000.1547717679.1550143142.1550154097.3; _jzqc=1; _jzqx=1.1547717679.1550154097.3.jzqsr=shenzhen%2Eqfang%2Ecom|jzqct=/sale/f2.jzqsr=shenzhen%2Eqfang%2Ecom|jzqct=/sale; _jzqb=1.1.10.1550154097.1; _qzjb=1.1550154096058.1.0.0.0'}

response = requests.get(url, headers=headers)

time.sleep(2)#延迟两秒时间

return response.text

3、#创建csv保存函数

def csv_data(item):

with open('fangwo_info.csv', 'a+', encoding='utf-8', newline='')as csvfile:#newline设置为''可以去点换行

writer = csv.writer(csvfile)

writer.writerow(item)

4、# 解析页面所需内容

def paser_info(url):

# 解析页面

html = spider_page(url)

selector = etree.HTML(html)#以构造器的形式返回

house_infos = selector.xpath('//*[@id="cycleListings"]/ul/li')

for house_info in house_infos:

name = house_info.xpath('./div[1]/p[1]/a/text()')[0].split(' ', 1)[0]

xiangq = house_info.xpath('./div[1]/p[1]/a/text()')[0].split(' ', 1)[1]

style = house_info.xpath('./div[1]/p[2]/span[2]/text()')[0]

area = house_info.xpath('./div[1]/p[2]/span[4]/text()')[0]

decotored = house_info.xpath('./div[1]/p[2]/span[6]/text()')[0]

louceng = house_info.xpath('./div[1]/p[2]/span[8]/text()')[0].strip()

chaoxiang = house_info.xpath('./div[1]/p[2]/span[10]/text()')[0]

total = house_info.xpath('./div[2]/span[1]/text()')[0]

price = house_info.xpath('./div[2]/p/text()')[0]

info = [name, style, area, decotored, louceng, chaoxiang, price, total, xiangq]

csv_data(info)

print("正在爬取", name)#编辑器里打开显示爬取

5、#创建主函数

def main():

# 添加csv标题头

info_title = ["名称", "户型", "面积", "装修", "楼层", "朝向", "售价", "总价/万", "详情"]

csv_data(info_title)

urls = ['https://shenzhen.qfang.com/sale/f%s' % x for x in range(1, 10)]

for url in urls:

paser_info(url)

6、# 调用函数运行

if __name__ == '__main__':

main()

最后爬取结果如下：