用requests + xpath 爬取小猪短租信息。
学习了xpath后,觉得这个工具真是非常的好用。
以下为完整代码:
import requests
from lxml import etree
import time
def getHTMLtext(url):
try:
r = requests.get(url,headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) App'
'leWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Sa'
'fari/537.36'})
r.raise_for_status()
r.encoding = r.apparent_encoding
return r.text
except:
return print("异常")
def parsehtml(html):
file = etree.HTML(html)
s = file.xpath('//*[@id="page_list"]/ul/li') #某页面下所有主题的xpath
#对每一个主题进行处理
with open(r'C:\Users\**\Desktop\xiaozhuduanzhu.txt','a',encoding='utf-8') as f:
for i in s:
title = i.xpath('./div[2]/div[2]/a/span/text()')[0]
price = i.xpath('./div[2]/div[1]/span/i/text()')[0] #标签内的信息
picture = i.xpath('./a/img/@lazy_src')[0].strip() #标签内的属性
scrible = i.xpath('./div[2]/div[2]/em/text()')[0]
f.write("{}\t{}\t{}\t{}\n".format(title,price,scrible,picture))
def main():
#只爬取5页
for i in range(1,6):
url = 'http://cd.xiaozhu.com/search-duanzufang-p{}-0/'.format(i)
html = getHTMLtext(url)
time.sleep(1) #爬取太快会禁止访问
parsehtml(html)
if __name__ == '__main__':
main()