# -*- coding: UTF-8 -*-
import requests
from lxml import etree
import sys
import re
import os,time
import shutil
from opencc import OpenCC
#pycharm add OpenCC and openccpy
import concurrent.futures
from bs4 import BeautifulSoup
from tkinter import filedialog
import json
import tkinter as tk
#down
#https://pypi.org/project/opencc-python-reimplemented/#files
#python setup.py install
#pip install opencc-python-reimplemented
class noveldl():
# 小说主地址，后接小说编号
req_url_base = 'http://www.jjwxc.net/onebook.php?novelid='
# 头文件，可用来登陆，cookie可在浏览器或者client.py中获取
headerss = {'cookie': '',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36'}
percent = 0
index = [] # 目录
titleindex = [] # 标题
Summary = [] # 内容提要
fillNum = '' # 章节填充位数
rollSign = [] # 卷标
rollSignPlace = [] # 卷标位置
state = '' # 繁简转换状态
href_list = [] # 章节链接
td = []
failInfo = []
titleInfo = [1, 1, 1]
def clear(self):
self.percent = 0
self.index = []
self.titleindex = []
self.Summary = []
self.fillNum = 0
self.rollSign = []
self.rollSignPlace = []
self.state = ''
self.href_list = []
self.td = []
self.failInfo = []
# 下载单章
def get_sin(self, l):
titleOrigin = l.split('=')
i = self.href_list.index(l)
cont = requests.get(l, headers=self.headerss).content
dot = etree.HTML(cont.decode("GB18030", "ignore").encode("utf-8", "ignore").decode('utf-8'))
# tex:正文
tex = dot.xpath("//html/body/table[@id='oneboolt']/tr[2]/td[1]/div[@class='noveltext']/text()")
# he:标题
he = dot.xpath("//html/body/table[@id='oneboolt']/tr[2]/td[1]/div[@class='noveltext']/div[2]/h2/text()")
# tex1:作话
tex1 = dot.xpath(
"//html/body/table[@id='oneboolt']/tr[2]/td[1]/div[@class='noveltext']/div[@class='readsmall']/text()")
# sign:作话位置
sign = dot.xpath("//*[@id='oneboolt']/tr[2]/td[1]/div/div[4]/@class")
title = ''
# 序号填充
if self.titleInfo[0] == '1':
title = str(titleOrigin[2]).zfill(self.fillNum) + "#"
# 章节名称
if self.titleInfo[1] == '1':
title = title + " " + self.titleindex[i].strip()
# 内容提要
if self.titleInfo[2] == '1':
title = title + " " + self.Summary[i].strip()
if self.state == 's':
title = OpenCC('t2s').convert(title)
elif self.state == 't':
title = OpenCC('s2t').convert(title)
if self.href_list[i] in self.rollSignPlace:
v = self.rollSign[self.rollSignPlace.index(l)]
if self.state == 's':
v = OpenCC('t2s').convert(self.rollSign[self.rollSignPlace.index(l)])
elif self.state == 't':
v = OpenCC('s2t').convert(self.rollSign[self.rollSignPlace.index(l)])
# 创建章节文件
fo = open("z" + str(titleOrigin[2].zfill(4)) + ".txt", 'w', encoding='utf-8')
# 写入卷标
if self.href_list[i] in self.rollSignPlace:
fo.write("\r\n\r\n" + v.rstrip() + '\r\n')
print("\r\n" + v + "\r\n")
fo.write(title + '\r\n')
# 写入标题
else:
fo.write("\r\n\r\n" + title + "\r\n")
if len(he) == 0:
self.failInfo.append(titleOrigin[2].zfill(self.fillNum))
fo.write('下载失败！')
else:
# 作话在文前的情况
if str(sign) == "['readsmall']":
for m in tex1: # 删除无用文字及多余空格空行
vv = re.sub('@无限好文，尽在晋江文学城', '', str(m))
v = re.sub(' +', ' ', vv).strip()
v = re.sub(' ', '', v)
if self.state == 's':
v = OpenCC('t2s').convert(v)
elif self.state == 't':
v = OpenCC('s2t').convert(v)
v = re.sub('作者有话要说：', '作者有话要说：\n', v)
if v != "": # 按行写入正文
fo.write(v + "\n")
if len(tex1) != 0:
fo.write("\n*\r\n")
for tn in tex:
vv = re.sub('@无限好文，尽在晋江文学城', '', str(tn))
v = re.sub(' +', ' ', vv).strip()
v = re.sub(' ', '', v)
if self.state == 's':
v = OpenCC('t2s').convert(v)
elif self.state == 't':
v = OpenCC('s2t').convert(v)
if v != "":
fo.write(v + "\n")
else: # 作话在文后的情况
for tn in tex:
vv = re.sub('@无限好文，尽在晋江文学城', '', str(tn))
v = re.sub(' +', ' ', vv).strip()
v = re.sub(' ', '', v)
if self.state == 's':
v = OpenCC('t2s').convert(v)
elif self.state == 't':
v = OpenCC('s2t').convert(v)
if v != "":
fo.write(v + "\n")
if len(tex1) != 0:
fo.write("\n*\r\n")
for m in tex1:
vv = re.sub('@无限好文，尽在晋江文学城', '', str(m))
v = re.sub(' +', ' ', vv).strip()
v = re.sub(' ', '', v)
if self.state == 's':
v = OpenCC('t2s').convert(v)
elif self.state == 't':
v = OpenCC('s2t').convert(v)
v = re.sub('作者有话要说：', '作者有话要说：\n', v)
if v != "":
fo.write(v + "\n")
fo.close()
self.percent += 1
def get_txt(self, txt_id, state, threadnum):
titlem = ''
intro = ''
ids = str(txt_id)
percent = 0
self.state = state
self.percent = 0
self.index = []
self.titleindex = []
self.Summary = []
self.fillNum = 0
self.rollSign = []
self.rollSignPlace = []
self.href_list = []
self.td = []
self.failInfo = []
# 获取文章网址
req_url = ids
# 通过cookie获取文章信息
res = requests.get(req_url, headers=self.headerss).content
# 对文章进行编码
ress = etree.HTML(res.decode("GB18030", "ignore").encode("utf-8", "ignore").decode('utf-8'))
# 获取文案
intro = ress.xpath("//html/body/table/tr/td[1]/div[2]/div[@id='novelintro']//text()")
# 获取标签
info = ress.xpath("string(/html/body/table[1]/tr/td[1]/div[3])")
infox = []
for i in range(1, 7):
infox.append(ress.xpath("string(/html/body/table[1]/tr/td[3]/div[2]/ul/li[" + str(i) + "])"))
# 获取标题
titlem = ress.xpath("//html/head/title/text()")
if self.state == 's':
titlem[0] = OpenCC('t2s').convert(titlem[0])
elif self.state == 't':
titlem[0] = OpenCC('s2t').convert(titlem[0])
print("网址：" + ids + "\r\n小说信息：" + str(titlem[0]) + "\r\n")
# 获取所有章节网址、标题、内容提要
self.td = ress.xpath('//*[@id="oneboolt"]//tr')
loc = []
for i in self.td:
u = i.xpath('./td[2]/span/div[1]/a/@href')
x = i.xpath('./td[2]/span/div[1]/a[1]/@rel')
if len(u) > 0:
self.href_list += u
v = i.xpath('./td[2]/span/div[1]/a')
v = etree.tostring(v[0], encoding="utf-8").decode().strip()
v = re.sub('</?\w+[^>]*>', '', v)
self.titleindex.append(v.strip())
v = i.xpath('./td[3]')
v = etree.tostring(v[0], encoding="utf-8").decode().strip()
v = re.sub('</?\w+[^>]*>', '', v)
v = re.sub(' ', '', v)
self.Summary.append(v.strip())
elif len(x) > 0:
self.href_list += x
v = i.xpath('./td[2]/span/div[1]/a')
v = etree.tostring(v[0], encoding="utf-8").decode().strip()
v = re.sub('</?\w+[^>]*>', '', v)
self.titleindex.append(v.strip())
v = i.xpath('./td[3]')
v = etree.tostring(v[0], encoding="utf-8").decode().strip()
v = re.sub('</?\w+[^>]*>', '', v)
v = re.sub(' ', '', v)
self.Summary.append(v.strip())
elif i.xpath('./td[2]/span/div[1]/span') != []:
loc.append(i.xpath('./td[1]/text()')[0].strip())
# 获取卷标名称
self.rollSign = ress.xpath("//*[@id='oneboolt']//tr/td/b[@class='volumnfont']")
# 获取卷标位置
self.rollSignPlace = ress.xpath(
"//*[@id='oneboolt']//tr/td/b/ancestor-or-self::tr/following-sibling::tr[1]/td[2]/span/div[1]/a[1]/@href")
self.rollSignPlace += ress.xpath(
"//*[@id='oneboolt']//tr/td/b/ancestor-or-self::tr/following-sibling::tr[1]/td[2]/span/div[1]/a[1]/@rel")
# 修改卷标格式
for rs in range(len(self.rollSign)):
self.rollSign[rs] = etree.tostring(self.rollSign[rs], encoding="utf-8").decode().strip()
self.rollSign[rs] = re.sub('</?\w+[^>]*>', '', self.rollSign[rs])
self.rollSign[rs] = "§ " + self.rollSign[rs] + " §"
section_ct = len(self.href_list)
print("可下载章节数：" + str(section_ct) + "\r\n")
if loc != []:
i = ""
for x in loc:
i = i + x + " "
print("被锁章节：" + i + "\r\n")
# fillNum：填充序号的长度，例如：若全文有1437章，则每章序号有四位，依次为0001、0002……
self.fillNum = len(str(len(self.td) - 4))
# 对标题进行操作，删除违规字符等
global ti
ti = str(titlem[0]).split('_')
ti = ti[0]
ti = re.sub('/', '_', ti)
ti = re.sub(r'\\', '_', ti)
ti = re.sub('\|', '_', ti)
ti = re.sub('\*', '', ti)
ti = re.sub('&', '&', ti)
xaut = ti.split('》')[1]
xauthref = ress.xpath("//*[@id='oneboolt']//h2/a/@href")[0]
xtitle = re.sub('《', '', ti.split('》')[0])
# 若文件名不想加编号，可以将这行删除
ti = ti + '.' + ids.split('=')[1]
ti = re.sub('\r', '', ti)
v = ""
# 打开小说文件写入小说相关信息
path = os.getcwd()
if os.path.exists(ti):
os.chdir(ti)
else:
os.mkdir(ti)
os.chdir(ti)
ppp = os.getcwd()
self.index = []
# 写入文章信息页
TOC = xtitle + '\n'
TOC += '作者：' + xaut + "\r\n"
# 生成目录文字
for l in self.href_list:
titleOrigin = l.split('=')
i = self.href_list.index(l)
#
title = str(titleOrigin[2]).zfill(self.fillNum) + " "
#
title = title + self.titleindex[i].strip() + " "
#
title = title + self.Summary[i].strip()
if self.state == 's':
title = OpenCC('t2s').convert(title)
elif self.state == 't':
title = OpenCC('s2t').convert(title)
if self.href_list[i] in self.rollSignPlace:
v = self.rollSign[self.rollSignPlace.index(l)]
if self.state == 's':
v = OpenCC('t2s').convert(self.rollSign[self.rollSignPlace.index(l)])
elif self.state == 't':
v = OpenCC('s2t').convert(self.rollSign[self.rollSignPlace.index(l)])
self.index.append(v)
self.index.append(title)
for ix in infox:
ix = ix.strip()
ix = re.sub('\r\n', '', ix)
ix = re.sub(' +', '', ix)
TOC += ix + "\r\n"
TOC += "文案：\r\n"
for nx in intro:
v = re.sub(' +', ' ', str(nx)).strip()
if self.state == 's':
v = OpenCC('t2s').convert(v)
elif self.state == 't':
v = OpenCC('s2t').convert(v)
if v != "":
TOC += v + "\n"
info = re.sub(' +', ' ', info).strip()
if self.state == 's':
info = OpenCC('t2s').convert(info)
elif self.state == 't':
info = OpenCC('s2t').convert(info)
info = re.sub('搜索关键字', '\r\n搜索关键字', info)
info = re.sub(' 一句话简介：', '一句话简介：', info)
info = re.sub('\r\n \r\n 立意：', '\r\n立意：', info)
TOC += info + "\n"
fo = open("TOC.txt", 'w', encoding='utf-8')
fo.write(TOC)
fo.close()
tlist = []
# 获取每一章内容
with concurrent.futures.ThreadPoolExecutor(max_workers=threadnum) as executor:
tlist = {executor.submit(self.get_sin, i): i for i in self.href_list}
for future in concurrent.futures.as_completed(tlist):
if self.percent < section_ct:
print('\r 下载进度：%d/%d' % (self.percent, section_ct), end='', flush=True)
print('\r 下载完成，总进度：%d/%d\r\n' % (self.percent, section_ct), end='', flush=True)
if self.failInfo != []:
self.failInfo.sort()
vs = ""
for ss in self.failInfo:
vs = vs + ss + "|"
print("\r\n未购买或加载失败章节：")
print(vs[:-1] + "\r\n")
# 整合
os.chdir(path)
f = open(ti + ".txt", 'w', encoding='utf-8')
filenames = os.listdir(ppp)
i = 0
for filename in filenames:
filepath = ppp + '\\' + filename
for line in open(filepath, encoding='utf-8', errors='ignore'):
f.writelines(line)
f.close()
shutil.rmtree(ppp)
print("\r\ntxt文件整合完成")
import codecs
# oldfile:UTF8文件的路径
# newfile:要保存的ANSI文件的路径
def convertUTF8ToANSI(oldfile, newfile):
fp_ansi = open(newfile, 'wb') # 转码后输出的文件
fp_utf8 = open(oldfile, 'rb') # 待转码的文件
data = ""
data = fp_utf8.read()
data = data.decode('utf-8') # 以二进制格式读入的数据需要先转码（转为内部码）才可以继续转换
data = data.encode('mbcs', errors='ignore') # 关键，将内部码再次编码
fp_ansi.write(data) # 写入文件
fp_ansi.close()
fp_utf8.close()
#import novel_login
# print('请输入cookie：')
# cookie=input()
# 此处为需要下载小说的编号，编号获取方法在上文中已经讲过，
#while 1:
#bookid = input('\r\n请输入晋江小说id号：\n')
def down():
url01='http://www.jjwxc.net/onebook.php?novelid='
num=url01+bookid
c = noveldl()
#state = input('\r\n文章内容：\r\n1、繁转简（输入s）\r\n2、简转繁（输入t）\r\n3、不变（直接按回车）\r\n')
state='3'
titleInfo = '1 1 1'
'''
titleInfo = input(
'\r\n请输入标题保存状态(序号 章节名称 内容提要)\r\n显示则输入1，不显示则输入0，数字之间用空格隔开\r\n例如：若只显示序号和内容提要，则输入[1 0 1](方括号不输入)\r\n若全部显示，可以直接按回车，若不显示标题，可以直接输入0\r\n若输入的数字个数小于3，则空缺的数字与最后输入的数字相同\r\n')
if titleInfo == '':
titleInfo = '1 1 1'
'''
titleInfo = titleInfo.split(' ')
while len(titleInfo) < 3:
titleInfo.append(titleInfo[len(titleInfo) - 1])
c.titleInfo = titleInfo
c.get_txt(num, state, 50)
convertUTF8ToANSI(ti + ".txt",ti +'-ansi.txt')
time.sleep(2)
os.remove(ti + ".txt")
#4335877 （99jie）
#https://github.com/7325156/jjwxcNovelCrawler
def bookidname():
global bookid
#file = filedialog.askopenfilename()
content = t1.get(0.0, 'end')
content2 = content.replace('\n', '')
bookid = content2
t1.delete(0.0, 'end')
t1.insert('end', bookid)
root=tk.Tk()
root.title('下载晋江小说免费- web: rogabet.xyz')
root.geometry()
l1=tk.Label(root,text='请输入小说ID号码 ( eg:3524551):')
l1.grid()
t1=tk.Text(root,width=40,height=1)
t1.grid()
b1=tk.Button(root,text="1.输入Book-Id-点我确认",width=20,command=bookidname)
b1.grid(row=2,column=0)
b2=tk.Button(root,text="2.开始下指定的晋江小说",width=20,command=down)
b2.grid(row=3,column=0)
def qc():
t1.delete(0.0,'end')
os.startfile('novel_login.exe')
b3=tk.Button(root,text="0.登录晋江--清除输入框",width=20,command=qc)
b3.grid(row=4,column=0)
root.mainloop()
663
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
