import requests
from lxml import etree
import time
import json
import re
# 书面链接有每本书的编码,可复制到这里修改
book_name = 130900
# https://doupocangqiong1.com/130900/
url = f'https://doupocangqiong1.com/'
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36",
}
def write_txt(html, title):
with open(f'{book}.txt', 'a+', encoding='utf-8') as f:
f.write(f'{title}\n\n')
f.write(f'{html}\n\n')
f.write('*' * 100)
html = requests.get(f'{url}{book_name}', headers=headers)
html.encoding = 'utf-8'
html = etree.HTML(html.text)
titles = html.xpath('/html/body/section/div[3]/div[2]/ul/li/a/@href')
# 获取书名
book = html.xpath('/html/body/section/div[1]/div[1]/div/div[1]/h1/text()')[0]
# print(book)
for i in titles:
try:
html = requests.get(f'{url}{i}', headers=headers, timeout=5)
except:
html = requests.get(f'{url}{i}', headers=headers,)
html.encoding = 'utf-8'
title = re.findall(r'<h1><a href="/\d+/\d+.html" title="(.*)">', html.text)[0]
cid = re.findall(r"cid:'(\d+)'}", html.text)[0]
data = {
'siteid': 0,
'bid': book_name,
'cid': cid,
}
result_url = 'https://doupocangqiong1.com/novelsearch/chapter/transcode.html'
try:
html = requests.post(result_url, headers=headers, data=data, timeout=5).json()
except:
html = requests.post(result_url, headers=headers, data=data,).json()
html = re.sub(r'</?\w+[^>]*>', '', html['info'])
# print(html)
print(title)
# time.sleep(0.5)
write_txt(html, title)
无难点内容,随手写的保存下,有时网络会请求超时,所以设置了第二次请求,,可能也是报错,但是懒得管了。。。。