爬取 小说

import os
import requests
from lxml import etree
import time


class Novel:
    def __init__(self):
        self.url = 'https://www.wxsy.net/novel/71254/'
        self.headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36 Edg/111.0.1661.54'
        }

    def get_data(self, url):
        resp = requests.get(url, headers=self.headers)
        if resp.status_code == 200:
            return resp.text
        else:
            return None

    def parse_data(self, response):
        html = etree.HTML(response)
        dds = html.xpath('//div[@class="pt-chapter-cont-detail full"]//a/@href')
        return dds

    def parse_url(self, parse):
        html = etree.HTML(parse)
        title = html.xpath('//h1[@class="lh100 size26 mb20"]/a/@title')[0].replace('?', '')
        content = html.xpath('//p[@class="content_detail"]/text()')
        return title, content

    def download(self, content):
        with open('./这些大人物其实都是我/'+content[0]+'.text', 'w', encoding='utf-8') as f:
            f.writelines(content[1])
            print(content[0]+'下载成功')


    def main(self):
        response = self.get_data(self.url)
        href = self.parse_data(response)
        os.mkdir('这些大人物其实都是我')
        for data in href:
            url = 'https://www.wxsy.net' + data
            parse = self.get_data(url)
            content = self.parse_url(parse)
            self.download(content)
            time.sleep(0.5)


if __name__ == '__main__':
    novel = Novel()
    novel.main()