网络小说下载python爬虫记录

# -*- coding:gbk -*-
import requests
from bs4 import BeautifulSoup
import urllib.parse
import codecs
import time

def write_txt(title, content):
    f = codecs.open('D:\\wushangshenwang.txt', 'a+')
    f.write(title)
    f.write('\n')  # 换行
    f.write('\n')  # 换行
    content = content.replace('\xa0','')
    f.write(content)
    f.write('\n')  # 换行
    f.write('\n')  # 换行
    f.close()

def proc_single_html(prefix,url):
    response = requests.get(prefix+url)
    print(prefix + url)
    response.encoding = 'gbk'
    bs_response = BeautifulSoup(response.text,'html.parser')
    title = bs_response.find(class_='book reader').find('h1').text
    content = bs_response.find(class_="showtxt").text
    write_txt(title, content)
    next_chapter = bs_response.find(class_='page_chapter')
    uls = next_chapter.ul
    lis = uls.findAll('li')

    for li in lis:
        if li.text == '下一章':
            a = li.find('a')
            if a != None:
                proc_single_html(prefix, a['href'])

                time.sleep(5)


proc_single_html('https://www.xxxx.lu','/book/61/11749.html')