1、目的
练习使用requests库和xpath
2、代码
# 0、导入所需要的包
import requests
from lxml import etree
import time
# 1、定制请求头并获取网页信息
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36",
"Referer": "http://www.mee.gov.cn/hjzl/sthjzk/zghjzkgb/index_1.shtml",
"Cookie": "_gscu_1697192173=83058333229jb885; gwdshare_firstime=1590330989156; wdcid=41a89784bf0d6d02; viewsid=02e888404ba1499f9568ca92c7181c86; _gscbrs_1697192173=1; wdses=003a2508f217103c; Hm_lvt_0f50400dd25408cef4f1afb556ccb34f=1609998583,1610070284; _gscs_1697192173=10070282bgiciw10|pv:8; wdlast=1610070326; Hm_lpvt_0f50400dd25408cef4f1afb556ccb34f=1610070326"
}
target_url = "http://www.mee.gov.cn/hjzl/sthjzk/zghjzkgb/index.shtml"
response = requests.get(target_url, headers=headers)
response.encoding = "utf-8"
text = response.text
# 2、使用xpath提取需要的信息
html = etree.HTML(text)
titles = html.xpath("//ul[@id='div']//a/text()")
file_urls = html.xpath("//ul[@id='div']//a/@href")
# 3、拼接得到完整的文件获取URL
full_urls = []
for url in file_urls:
full_url = "http://www.mee.gov.cn/hjzl/sthjzk/zghjzkgb" + url[1:]
full_urls.append(full_url)
# 4、批量下载并将文件保存在指定目录
for i in range(len(titles)):
path = r"C:\Users\Desktop\阅读文献\环境公报\\"
filename = path + titles[i] + ".pdf"
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36",
"Referer": "http://www.mee.gov.cn/hjzl/sthjzk/zghjzkgb/index.shtml",
}
target_url = full_urls[i]
response = requests.get(target_url, headers=headers)
# 注意指定文件是以"wb",也就是二进制方式写入
fp = open(filename, "wb")
fp.write(response.content)
fp.close()
time.sleep(5)