爬取政府工作报告(服从爬虫协议规范)
import requests
import datetime
import sys
from lxml import etree
import re
#输入任意字符
inpu = input()
#判断使用者输入是否为整数
if inpu.isdigit()==True:
input_year = int(inpu)
else:
print("输入类型不符请重试!")
sys.exit()
#当前年份
max_year = datetime.datetime.now().year
print(type(max_year))
#设定url和headers
if input_year in range(2014,max_year):
url = 'http://www.gov.cn/guowuyuan/'+str(input_year)+'zfgzbg.htm'
else:
url = 'http://www.gov.cn/guowuyuan/zfgzbg.htm'
print(url)
headers = {
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.135 Safari/537.36'
}
#发送请求
response = requests.get(url = url,headers = headers)
response.encoding = 'utf-8'
html = response.text
html_tree = etree.HTML(html)
result = html_tree.xpath('//div[@id="conlun2_box_text"]/descendant::text()')
print(type(result))
for i in result:
print(i)
with open('spider.txt','a',encoding = 'utf-8') as file:
file.write('\n'.join(result))