python 爬取起点中文网的小说(学习记录)

Time:2020-06-12
程序实现功能:实现从起点中文网爬取小说内容;
（说明:只做学习交流）

import requests, chardet
from lxml import html
from bs4 import BeautifulSoup
from urllib.parse import urlencode
import json
import jsonpath
import urllib.request
headers = {'User-Agent':"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.10240",
               'Referer': 'https://book.qidian.com/info/3144877',
               'Cookie': '_csrfToken=BXnzDKmnJamNAgLu4O3GknYVL2YuNX5EE86tTBAm; newstatisticUUID=1564467217_1193332262; qdrs=0%7C3%7C0%7C0%7C1; showSectionCommentGuide=1; qdgd=1; lrbc=1013637116%7C436231358%7C0%2C1003541158%7C309402995%7C0; rcr=1013637116%2C1003541158; bc=1003541158%2C1013637116; e1=%7B%22pid%22%3A%22qd_P_limitfree%22%2C%22eid%22%3A%22qd_E01%22%2C%22l1%22%3A4%7D; e2=%7B%22pid%22%3A%22qd_P_free%22%2C%22eid%22%3A%22qd_A18%22%2C%22l1%22%3A3%7D'
    }
def get_url():
    data = {
        '_csrfToken': '4VYymAMRlhyz1IhPAAWDoWSCfArlgeU0d3RozEwH',
        'bookId': '1020943839'
    }
    url = 'https://read.qidian.com/ajax/book/category?' + urlencode(data)
    return url
#/*
#          数据获取部分
# */
def getdata():
    response = requests.get(get_url(), headers=headers)
    content = response.content
    encoding = chardet.detect(content).get('encoding')  # 编码格式
    content = content.decode(encoding, 'ignore')  # 忽略非法字符,转换16进制
    return content
def get_url_data():
    unicodestr = json.loads(getdata())
    TextUrl = jsonpath.jsonpath(unicodestr, "$..cU")
    return TextUrl
def get_title_data():
    unicodestr = json.loads(getdata())
    TitleUrl = jsonpath.jsonpath(unicodestr, "$..cN")
    return TitleUrl
#/*
#           数据下载部分
# */
def Download():
    Name=input("请输入文件txt名称：")
    print("数据开始下载......下载中")
    URL=[]
    Title=[]
    for i in get_url_data():
        URL.append("https://read.qidian.com/chapter/"+i)
    for j in get_title_data():
        Title.append(j)
    for Biao,title in enumerate(URL):
        with open(Name, 'a+', encoding='utf-8') as f:
            response = urllib.request.urlopen(title)
            html = response.read().decode('UTF-8')
            Div = BeautifulSoup(html, 'html.parser')
            GetText = Div.find_all('div', class_='read-content j_readContent')  # 过滤class为X的标签
            ui = GetText[0].text.replace('\xa0', '\n\n')
            data=Title[Biao]+ui
            f.write(data+ '\n')
    print("数据下载完毕!!!")
if __name__ == "__main__":
    Download()

只做记录学习
（qq:九七二四三九三二九，有不懂的可以问我，一起学习）