python爬虫（小说）

导入库

import requests
from lxml import etree
import os

获取小说目录url

def catalog(url):
    """
    获取所有目录url
    :param url:网址
    :return:目录url
    """
    try:
        rq = requests.get(url = url,headers = HEADERS,timeout = 10)
        tree = etree.HTML(rq.text)
     	book_title = tree.xpath('//div[@class = "mu_h1"]/h1/text')
        mulu_list = tree.xpath('//ul[@class = "mulu_list"]/li')
        mulu_url = [url+''.join(i.xpath('./a/@href')) for i in mulu_list]
        return mulu_url
    except:
        print("catalog出错！")

解析网页数据获取文本内容

def mulu_content(mulu_url):
    """
    解析小说内容
    :param mulu_url:目录url
    :return:无
    """
    try:
        response = requests.get(url = mulu_url,headers = HEADERS,timeout = 5)
        response.encoding = response.apparent_encoding
        tree = etree.HTML(response.text)
        content = tree.xpath('//div[@id = "htmlContent"]/text()') # 内容
        content = "".join(content).strip().replace("  ","")
        title = tree.xpath('//div[@class = "h1title"]/h1/text()') # 标题
        title = "".join(title).strip()
        print(content,title)
        if not os.path.exists(r'./文件夹名称'):
            os.mkdir(r'./文件夹名称')
        if not os.path.exists(f'./文件夹名称/{title}.txt'):
            with open(f'./文件夹名称/{title}.txt','w',encoding = 'utf-8') as f:
                f.write(content)
                print(title+"采集完成！")
        else:
            print(title+'本章已采集！')
        print("小说采集完成！")
    except:
        print('mulu_content出错！')

主函数

if __name__  ==  '__main__':
    url = 'https://www.duquanben.com/xiaoshuo/13/13695/'
    HEADERS = {
        "user-agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36 SLBrowser/7.0.0.8031 SLBChan/30"
    }
   for mulu_url in catalog(url):
		mulu_content(mulu_url)