第一次用爬虫批量下载小说章节

import requests#应用库
from lxml import etree#引用库
import os#引用库
if not os.path.exists('./终极斗罗/'):#创建终极斗罗文件夹
    os.makedirs('./终极斗罗/')
url='http://www.biquw.com/book/16583/'#获取小说网站链接
headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36 Edg/88.0.705.56'}#进行UA伪装
r=requests.get(url=url,headers=headers).text.encode(encoding='ISO-8859-1').decode()#获取响应数据
response=etree.HTML(r)#解析HTML文档
list=response.xpath('//li/a/@href');newlist=[]#用xpath获取每个章节的链接
for url in list:#遍历每个章节并拼接
    new='http://www.biquw.com/book/16583/'+url
    r1=requests.get(url=new,headers=headers).text.encode(encoding='ISO-8859-1').decode()#获取响应数据
    r2 = etree.HTML(r1, etree.HTMLParser())#解析HTML文档
    text = r2.xpath('//div[@id="htmlContent"]/text()');word=''#用xpath获取文本内容
    for i in text:#遍历列表拼接其内容
        word+=i
    name = r2.xpath('//title/text()') # 用xpath匹配文本内容的名字
    for name1 in name:#遍历章节名字
        pass
    with open('./终极斗罗/'+name1+'.txt','w',encoding='utf-8') as fp:#永久化储存
        fp.write(word)

跟批量爬取图片类似。
有不足的地方请大佬指点。