平安实习—爬取环保部近15年的生态环境质量公报（爬虫练习）

1、目的

练习使用requests库和xpath

2、代码

# 0、导入所需要的包
import requests
from lxml import etree
import time

# 1、定制请求头并获取网页信息
headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36",
            "Referer": "http://www.mee.gov.cn/hjzl/sthjzk/zghjzkgb/index_1.shtml",
            "Cookie": "_gscu_1697192173=83058333229jb885; gwdshare_firstime=1590330989156; wdcid=41a89784bf0d6d02; viewsid=02e888404ba1499f9568ca92c7181c86; _gscbrs_1697192173=1; wdses=003a2508f217103c; Hm_lvt_0f50400dd25408cef4f1afb556ccb34f=1609998583,1610070284; _gscs_1697192173=10070282bgiciw10|pv:8; wdlast=1610070326; Hm_lpvt_0f50400dd25408cef4f1afb556ccb34f=1610070326"
}
target_url = "http://www.mee.gov.cn/hjzl/sthjzk/zghjzkgb/index.shtml"
response = requests.get(target_url, headers=headers)
response.encoding = "utf-8"
text = response.text

# 2、使用xpath提取需要的信息
html = etree.HTML(text)
titles = html.xpath("//ul[@id='div']//a/text()")
file_urls = html.xpath("//ul[@id='div']//a/@href")

# 3、拼接得到完整的文件获取URL
full_urls = []
for url in file_urls:
    full_url = "http://www.mee.gov.cn/hjzl/sthjzk/zghjzkgb" + url[1:]
    full_urls.append(full_url)

# 4、批量下载并将文件保存在指定目录
for i in range(len(titles)):
    path = r"C:\Users\Desktop\阅读文献\环境公报\\"
    filename = path + titles[i] + ".pdf"
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36",
        "Referer": "http://www.mee.gov.cn/hjzl/sthjzk/zghjzkgb/index.shtml",
    }
    target_url = full_urls[i]
    response = requests.get(target_url, headers=headers)
    # 注意指定文件是以"wb"，也就是二进制方式写入
    fp = open(filename, "wb")
    fp.write(response.content)
    fp.close()
    time.sleep(5)

3、爬取结果