使用python爬取小说（剑来）

地址：

http://www.jianlaixiaoshuo.com/

代码实现：

#导入模块
import requests
from pyquery import PyQuery as pq

headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Safari/537.36"
    }
#得到每章网址
response = requests.get("http://www.jianlaixiaoshuo.com/",headers=headers)
response.encoding = "utf8"
dob = pq(response.text)
links = dob("dl > dd a")

for link in links.items():

    response = requests.get("http://www.jianlaixiaoshuo.com" + link.attr.href,headers=headers)
    response.encoding="utf8"
    doc = pq(response.text)
    title = doc("#BookCon > h1").text()
    print(title + "下载完成")
    content = doc("#BookText").text()

    with open("2.txt",mode="a+",encoding="utf-8") as f:
        f.write(title)
        f.write(content)
        f.write("\n")