小说下载（在线阅读）

今天来说一下，小说下载。

以《人世间》为例：

网址：url = "http://www.loying.cc/book/renshijian/"

1.导入要用到的模块：

import os.path
import requests
import parsel
from urllib.parse import urljoin

2.网站反馈信息：

def get_response(url):
    headers = {
        'referer': url.split('//')[0]+"//"+url.split('//')[-1].split('/')[0],
        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36'}
    response = requests.get(url=url,headers=headers,timeout=30)
    return response

3.网页解析，得到Selector类

def get_selector(url):
    response = get_response(url)
    response.encoding = response.apparent_encoding
    selector = parsel.Selector(response.text)
    return selector

4.通过selector 得到 小说名称，作者、章节列表等内容：

名称：novel_name = selector.css('.media-heading.tc.title').css('h3').css('::text').get()

作者：novel_author = selector.css('.media-body').css('p').css('::text').get()

章节列表：
lists =[]
while True:
    list0 = selector.css('.book-list').css('a')
    lists = lists + list0
    print(len(lists))
    if selector.css('.listpage .right a[href]'):
        link_ = selector.css('.listpage .right a::attr(href)').get()
        link = urljoin(url,link_)
        selector = get_selector(link)
    else:
        break

5.单章内容下载：

def down_text(item):
    chapter_title = item.css('::text').get()
    chapter_link = urljoin(url,item.css('::attr(href)').get())
    print(chapter_title,chapter_link)
    selector = get_selector(chapter_link)
    content = selector.css('#nr1').css('p').css('::text').getall()
    content = [item.strip() for item in content if item.strip() != '']
    while True:
        next_page = selector.css('.read_nav.reader-bottom').css('a[href]')
        chapter_link_next = [urljoin(chapter_link,item.css('::attr(href)').get()) for item in next_page if item.css('::text').get() == '下一页']
        if chapter_link_next == []:
            break
        selector = get_selector(chapter_link_next[0])
        content0 = selector.css('.content').css('p').css('::text').getall()
        content0 = [item.strip() for item in content0 if item.strip() != '']
        content0 = content0[1:-1]
        content = content + content0

    content = [item for item in content if item != '（本章未完，请点击下一页继续阅读）']
    content = "\n".join(content)
    with open(novel_name+".txt",'a',encoding='utf-8') as fw:
        fw.write('《{}》\n\n{}\n\n'.format(chapter_title,content))
    print('.',end='')

6.逐章下载，追加方式保存文件。

7.完整代码。

import os.path
import requests
import parsel
from urllib.parse import urljoin

def get_response(url):
    headers = {
        'referer': url.split('//')[0]+"//"+url.split('//')[-1].split('/')[0],
        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36'}
    response = requests.get(url=url,headers=headers,timeout=30)
    return response

def get_selector(url):
    response = get_response(url)
    response.encoding = response.apparent_encoding
    selector = parsel.Selector(response.text)
    return selector

def down_text(item):
    chapter_title = item.css('::text').get()
    chapter_link = urljoin(url,item.css('::attr(href)').get())
    print(chapter_title,chapter_link)
    selector = get_selector(chapter_link)
    content = selector.css('#nr1').css('p').css('::text').getall()
    content = [item.strip() for item in content if item.strip() != '']
    while True:
        next_page = selector.css('.read_nav.reader-bottom').css('a[href]')
        chapter_link_next = [urljoin(chapter_link,item.css('::attr(href)').get()) for item in next_page if item.css('::text').get() == '下一页']
        if chapter_link_next == []:
            break
        selector = get_selector(chapter_link_next[0])
        content0 = selector.css('.content').css('p').css('::text').getall()
        content0 = [item.strip() for item in content0 if item.strip() != '']
        content0 = content0[1:-1]
        content = content + content0

    content = [item for item in content if item != '（本章未完，请点击下一页继续阅读）']
    content = "\n".join(content)
    with open(novel_name+".txt",'a',encoding='utf-8') as fw:
        fw.write('《{}》\n\n{}\n\n'.format(chapter_title,content))
    print('.',end='')


url = "http://www.loying.cc/book/renshijian/"
selector = get_selector(url)
novel_name = selector.css('.media-heading.tc.title').css('h3').css('::text').get()
print(novel_name)
novel_author = selector.css('.media-body').css('p').css('::text').get()
print(novel_author)
with open(novel_name + ".txt", 'w', encoding='utf-8') as fw:
    fw.write('《{}》\n\n{}\n\n'.format(novel_name, novel_author))
lists =[]
while True:
    list0 = selector.css('.book-list').css('a')
    lists = lists + list0
    print(len(lists))
    if selector.css('.listpage .right a[href]'):
        link_ = selector.css('.listpage .right a::attr(href)').get()
        link = urljoin(url,link_)
        selector = get_selector(link)
    else:
        break
temp_path = "temp"
if not os.path.exists(temp_path):
    os.makedirs(temp_path)
else:
    for file in os.listdir(temp_path):
        os.remove(os.path.join(temp_path,file))
for item in lists:
    down_text(item)