python 爬取百度搜索结果url

简单的爬取百度搜索结果url
先用了requests库来访问百度，再通过xpath来提取搜索后的结果

import requests
from lxml import etree

for i in range(0,100,10):#通过for循环来实现翻页
	url = 'https://www.baidu.com/s?wd=爬虫&pn=' + str(i)+ '&oq=爬虫'
	headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3100.0 Safari/537.36'}
	r = requests.get(url, headers = headers ,timeout = 5).text
	
	a = etree.HTML(r)
	xpath = a.xpath('//*[@class="t"]/a/@href')#用xpath来提取百度搜索结果url链接
	for i in xpath:
		url = i 
		try:
			r = requests.get(url,headers = headers , timeout = 5)
			print(r.url)
		except:
			print('无法链接')