采集网页静态文本内容

爬取政府工作报告（服从爬虫协议规范）

import requests
import datetime
import sys
from lxml import etree
import re

#输入任意字符
inpu = input()
#判断使用者输入是否为整数
if inpu.isdigit()==True:
    input_year = int(inpu)
else:
    print("输入类型不符请重试！")
    sys.exit()

#当前年份
max_year = datetime.datetime.now().year
print(type(max_year))

#设定url和headers
if input_year in range(2014,max_year):
    url = 'http://www.gov.cn/guowuyuan/'+str(input_year)+'zfgzbg.htm'
else:
    url = 'http://www.gov.cn/guowuyuan/zfgzbg.htm'
print(url)

headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.135 Safari/537.36'
}

#发送请求
response = requests.get(url = url,headers = headers)
response.encoding = 'utf-8'
html = response.text
html_tree = etree.HTML(html)
result = html_tree.xpath('//div[@id="conlun2_box_text"]/descendant::text()')
print(type(result))
for i in result:
    print(i)
with open('spider.txt','a',encoding = 'utf-8') as file:
    file.write('\n'.join(result))