不多说,先上勘误代码
# 链家经纪人爬虫实现
import requests
import csv
import time
from lxml import etree
def csv_writer(item):
with open('csvFile/lianjia_broker.csv', 'a', encoding='gbk') as csvfile:
writer = csv.writer(csvfile)
writer.writerow(item)
def spider(list_url):
headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
'AppleWebKit/537.36 (KHTML, like Gecko) '
'Chrome/72.0.3626.96 Safari/537.36'}
response = requests.get(list_url, headers=headers)
time.sleep(5)
sel = etree.HTML(response.text)
agent_list = sel.xpath('//li[@class="pictext flexbox '
'box_center_v lazyload_ulog"]/div'
'[@class="post_ulog agent_index"]')
for agent in agent_list:
agent_name = agent.xpath('div[2]/div[1]/span[1]/a/text()')[0]
agent_region = agent.xpath('div[2]/div[2]/span[1]/text()')[0]
item = [agent_name, agent_region]
print('正在抓取:', agent_name)
csv_writer(item)
if __name__ == '__main__':
for i in range(1, 101):
url = ('https://m.lianjia.com/bj/jingjiren/?'
'page_size=15&_t=1&offset=' + str(i * 15))
spider(url)
主要修改的代码如下:
# 原书代码
agent_list = sel.xpath('//li[@class="pictext flexbox box_center_v"]')
# 修改的代码
agent_list = sel.xpath('//li[@class="pictext flexbox '
'box_center_v lazyload_ulog"]/div'
'[@class="post_ulog agent_index"]')
一开始是网页中的li的class属性名称变化,通过查询网页修改class之后,出现IndexError: list index out of range报错。是因为agent_name = agent.xpath('div[2]/div[1]/span[1]/a/text()')[0]这里传入的是div[2],而agent是一个li,所以找不到name所在的<a></a>标签,故也没有返回list对象。
刚开始学Python,可能术语表述不当。溜了溜了。
Best wishes.