-
爬取https://www.duquanben.com/小说实现断线之后继续从上次断开的章节下载。
-
sqlite安装
-
SQLite Download Page 下载 sqlite-dll-win32-x86-3350500.zip sqlite-tools-win32-x86-3350500.zip 解压到目录,添加环境变量。
-
创建databases,tables。
C:\sqlite3 test.db SQLite version 3.35.5 2021-04-19 18:32:05 Enter ".help" for usage hints. sqlite> CREATE TABLE booklist( ...> ID INT PRIMARY KEY NOT NULL, ...> title str NOT NULL, ...> url str NOT NULL, ...> status str NOT NULL ...> );
-
-
模块安装
- pip install sqlite3 and pyquery
-
代码
-
# -*- coding:utf-8 -*- import requests from pyquery import PyQuery as pq import time import sqlite3 header = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.72 Safari/537.36" } url='https://www.duquanben.com/xiaoshuo/10/10413/' name = """九阴九阳之阴阳神功""" class crawle(object): def __init__(self, *args): self.header = header self.url = url self.name = name ''' get小说主页面,保存html ''' def down_book_list(self): try: req=requests.get(url=self.url,headers=self.header,timeout=20) encodings = requests.utils.get_encodings_from_content(req.text) encoding = req.apparent_encoding encode_content = req.content.decode(encoding, 'replace') with open('booklist.html','w',encoding='utf-8') as f: f.write(encode_content) f.close() except Exception as e: print(e) ''' 读取保存的html,通过pyquery读取到小说每一章的url,存入数据库,默认开始所有的状态都没none ID title url status 1460|第1461章 万龙巢穴(1)|2097128.html|none 1461|第1462章 万龙巢穴(2)|2097129.html|none 1462|第1463章 万龙巢穴(3)|2097130.html|none 1463|第1464章 万龙巢穴(4)|2097131.html|none 1464|第1465章 万龙巢穴(5)|2097132.html|none 1465|第1466章 万龙巢穴(6)|2097133.html|none 1466|第1467章 师徒大战(1)|2097134.html|none 1467|第1468章 师徒大战(2)|2097135.html|none 1468|第1469章 师徒大战(3)|2097136.html|none 1469|第1470章 师徒大战(4)|2097137.html|none ''' def save_book_list(self): with open("booklist.html","r",encoding="utf-8")as f: content = f.read() doc = pq(content) items=doc('.mulu_list li').items() status = {} conn = sqlite3.connect("xiaoshuo.db") cursor = conn.cursor() id = 0 for each in items: title_ = each.text() url_ = each.find('a').attr('href') cursor.execute("INSERT INTO booklist (id,title,url,status) VALUES ('{0}','{1}','{2}','none')".format(id,title_,url_)) id = id + 1 conn.commit() conn.close() ''' 下载小说,下载成功的章节会改变status为True,暂停之后下次下载会从第一个none的节点开始下载。 sqlite> select * from booklist ; 0|第1章 孔家逆子|2095666.html|True 1|第2章 红光乍现|2095667.html|True 2|第3章 炼精化气|2095668.html|True 3|第4章 九阴神爪|2095669.html|True 4|第5章 华家月儿|2095670.html|True 5|第6章 五禽五行|2095671.html|none 6|第7章 阶下之囚|2095672.html|none 7|第8章 五行均衡|2095673.html|none 8|第9章 月儿授艺|2095674.html|none 9|第10章 周身穴窍|2095675.html|none ''' def down_book(self): conn = sqlite3.connect("xiaoshuo.db") cursor = conn.cursor() cursor.execute("select count(*) from booklist where status = 'none'") sum_num = cursor.fetchall()[0][0] cursor.execute("select * from booklist where status = 'none' limit 1") id = int(cursor.fetchall()[0][0]) for i in range(sum_num): time.sleep(3) cursor.execute("select * from booklist where ID = {0}".format(id)) data = cursor.fetchall() try: url = self.url + data[0][2] req=requests.get(url=url,headers=header,timeout=20) encodings = requests.utils.get_encodings_from_content(req.text) encoding = req.apparent_encoding encode_content = req.content.decode(encoding, 'replace') doc=pq(encode_content) item=doc('.contentbox').text() with open(self.name,'a',encoding='utf-8') as f: f.write(data[0][1]+'\n') f.write(item) cursor.execute("UPDATE booklist SET status = ? where id = ?",("True","{0}".format(id))) print("download sucess {0} {1}".format(data[0][1],url)) conn.commit() id = id + 1 except Exception as e: print(e) conn.close() if __name__=='__main__': ''' 初始化,判断是否是第一次执行,失败后再执行一次继续下载。 ''' start = crawle(header,url,name) conn = sqlite3.connect("xiaoshuo.db") cursor = conn.cursor() cursor.execute("select count(*) from booklist where status = 'none'") sum_num = cursor.fetchall()[0][0] if sum_num: conn.close() start.down_book() else: conn.close() start.down_book_list() start.save_book_list() start.down_book() conn.close()