import requests
from lxml import etree
import json
class QiubaiSpider:
def __init__(self):
self.url_temp="https://www.qiushibaike.com/8hr/page/{}/"
self.headers = {"User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36"}
def get_url_list(self):
url_list=[self.url_temp.format(i) for i in range(1,14)]
return url_list
def parse_url(self,url):
response = requests.get(url, headers= self.headers)
return response.content.decode()
def get_content_list(self,html_str):
html = etree.HTML(html_str)
#1. 分组
div_list = html.xpath("//div[@id='content-left']/div")
content_list = []
for div in div_list :
item = {}
item['author_name']=div.xpath(".//h2//text()")[0].strip() if len(div.xpath(".//h2//text()"))>0 else None
item["content"] = div.xpath(".//div[@class='content']/span/text()")
item['content']=[i.strip() for i in item['content']]
item['status_vote'] = div.xpath("//span[@class='stats-vote']/i/text()")
item['status_vote'] = item['status_vote'][0] if len(item['status_vote'])>0 else None
content_list.append(item)
return content_list
def save_content_list(self,content_list):
with open("qibai.txt",'a',encoding='utf-8') as f:
for content in content_list:
f.write(json.dumps(content,ensure_ascii=False))
f.write("\n")
print ("save successfully")
def run(self):
#1.根据url地址的规律,构造url list
url_list = self.get_url_list()
#2. 发送请求,获取响应
for url in url_list:
html_str = self.parse_url(url)
content_list=self.get_content_list(html_str) #3.提取数据
self.save_content_list(content_list) #4.保存
if __name__ == '__main__':
qiubai= QiubaiSpider()
qiubai.run()