最近又重新爬虫,遇到一个之前没注意的问题,就是request返回对象的问题
from lxml import etree
currentAllInfo = []
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:53.0) Gecko/20100101 Firefox/53.0'}
sourceHtml = requests.get(url, headers=headers)//sourceHtml对象类型为response,不是 htmlresponse,因此需要转换
selector = etree.HTML(sourceHtml)
titles = selector.xpath('//dl[@class="blog_list clearfix"]//dd')
singlePageInfo = {};
for vs in titles:
info = vs.xpath('h3[@class="tracking-ad"]/a/text()')
print "标题:" + info[0]
singlePageInfo['title'] = info[0]
time = vs.xpath('div[@class="blog_list_b clearfix"]/div[@class="blog_list_b_r fr"]/label/text()')
print "时间:" + time[0]
singlePageInfo['time'] = time[0]
readCount = vs.xpath('div[@class="blog_list_b clearfix"]/div[@class="blog_list_b_r fr"]/span/em/text()')
print "阅读次数:" + readCount[0]
currentAllInfo.append(singlePageInfo)
print currentAllInfo