小说爬虫之幻月书院requests和re库

刚学了正则表达式，直接用上爬虫来试试，不多说直接上代码：

#!/usr/bin/python3
import requests
import re


class HuanYue:
    def __init__(self):
        self.url = "http://www.huanyue123.com/book/7/7717/"
        self.headers = {"user-agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.96 Safari/537.36"}

    def parse(self, parse_url):
        response = requests.get(url=parse_url, headers=self.headers).content.decode('gbk')
        html = re.compile('<div id="htmlContent" class="contentbox clear"><div style=".*?" ><a href=".*?" style=".*?">.*?</a></div>(.*?)</div>', re.S).findall(response)
        # print(html[0].replace('&nbsp;&nbsp;&nbsp;&nbsp;', '    ').replace('<br />', ''))
        txt = html[0].replace('&nbsp;&nbsp;&nbsp;&nbsp;', '    ').replace('<br />', '')
        return txt

    def write(self, txt):
        with open('C:\\Users\\cj\\Desktop\\小说.txt', 'a+', encoding='utf8') as f:
            f.write(txt)
            f.close()

    def run(self):
        req = requests.get(url=self.url, headers=self.headers).content.decode('gbk')
        text = re.compile('<li><a href="(.*?)">(.*?)</a></li>', re.S).findall(req)
        for url in text:
            parse_url = url[0]
            title = url[1]
            print('下载：', title)
            print('下载url：', parse_url)
            txt = self.parse(parse_url)
            self.write(txt)
            print(title, '下载完了！')


if __name__ == '__main__':
    t1 = HuanYue()
    t1.run()

以上是随便找了一个小说进行爬取