第一次写python爬虫

第一次写python爬虫

爬取58二手房网站

import requests
from lxml import etree

获取页面源码数据

url = ‘https://su.58.com/ershoufang/?utm_source=market&spm=u-2d2yxv86y3v43nkddh1.BDPCPZ_BT&PGTID=0d100000-0000-5181-2fa5-ee4c1abecbb9&ClickID=3’ \

headers = {
‘User-Agent’:‘Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.26 Safari/537.36 Core/1.63.6821.400 QQBrowser/10.3.3040.400’

}
page_text = requests.get(url=url,headers=headers).text

#实例化etree对象并将页面源码数据加载到对象中
tree = etree.HTML(page_text)
li_list = tree.xpath(’//ul[@class=“huose-list-wrap”]/li’)
all_data_list = []
for li in li_list:
title = li.xpath(’.//div[@class=“list-info”]/h2/a/text()’)[0]
detail_url = li.xpath(’.//div[@class=“list-info”]/h2/a/@href’)[0]
if not ‘https:’ in detail_url:
detail_url = ‘https:’+detail_url
price = li.xpath(’.//ul[@class=“price”]/p//text()’)
price = ‘’.join(price)

# 对详情页发起请求，获取页面数据
detail_page_text = requests.get(url=detail_url,headers=headers).text
tree = etree.HTML(detail_page_text)
desc = tree.xpath('.//div[@class="general-item-wrap""]//text()')
desc = ''.join(desc).strip("\n\b\t")

dic = {
    'title':title,
    'price':price,
    'desc':desc
}

all_data_list.append(dic)

print(all_data_list)

#爬取成功但是没数据