Python网络爬虫：下载小说的正确姿势

爬取网站新笔趣阁的小说诡秘之主

源代码：

from bs4 import BeautifulSoup
from tqdm import tqdm
import requests


def ask_url(url):
    try:
        r = requests.get(url, timeout=30)
        r.raise_for_status()
        r.encoding = 'utf-8'
        return r.text
    except:
        return ""


def get_data(url):
    html = ask_url(url)
    soup = BeautifulSoup(html, 'html.parser')
    content = soup.find('div', id="content")
    content = content.text.strip().split('\xa0' * 4)
    return content


if __name__ == '__main__':
    chapter_urls = []
    chapter_names = []
    url = 'https://www.xsbiquge.com/15_15338/'
    base_url = 'https://www.xsbiquge.com/'

    html = ask_url(url)
    soup = BeautifulSoup(html, 'html.parser')
    chapters = soup.find('div', id="list")
    chapters = chapters.find_all('a')

    for chapter in chapters:
        chapter_url = base_url + chapter.attrs['href']
        chapter_urls.append(chapter_url)
        chapter_names.append(chapter.string)

    with open("诡秘之主.txt", 'a', encoding='utf-8') as f:
        for i in tqdm(range(len(chapter_urls))):
            f.write(chapter_names[i])
            f.write('\n')
            content = get_data(chapter_urls[i])
            f.write('\n'.join(content))
            f.write('\n')
    
    # print(len(chapter_names))
    # print(len(chapter_urls))
    # for name in chapter_names:
    #     print(name)

    # for url in chapter_urls:
    #     print(url)

爬取结果截图：