python3 Requests+Sqlite+Pyquery断点下载小说爬虫

​​

• ​​爬取https://www.duquanben.com/小说实现断线之后继续从上次断开的章节下载。

• sqlite安装

  • SQLite Download Page 下载 sqlite-dll-win32-x86-3350500.zip sqlite-tools-win32-x86-3350500.zip 解压到目录，添加环境变量。

  • 创建databases，tables。

    C:\sqlite3 test.db 
    SQLite version 3.35.5 2021-04-19 18:32:05
    Enter ".help" for usage hints.
    sqlite> CREATE TABLE booklist(
       ...>    ID INT PRIMARY KEY     NOT NULL,
       ...>    title           str    NOT NULL,
       ...>    url            str     NOT NULL,
       ...>    status         str     NOT NULL
       ...> );

• 模块安装

  • pip install  sqlite3 and pyquery

• 代码

• # -*- coding:utf-8 -*-
  import requests
  from pyquery import PyQuery as pq
  import time
  import sqlite3
  header = {
  	"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.72 Safari/537.36"
  }
  url='https://www.duquanben.com/xiaoshuo/10/10413/'
  name = """九阴九阳之阴阳神功"""
  
  class crawle(object):
      def __init__(self, *args):
          self.header = header
          self.url = url
          self.name = name
      ''' get小说主页面，保存html '''      
      def down_book_list(self):
          try:
              req=requests.get(url=self.url,headers=self.header,timeout=20)
              encodings = requests.utils.get_encodings_from_content(req.text)
              encoding = req.apparent_encoding
              encode_content = req.content.decode(encoding, 'replace')
              with open('booklist.html','w',encoding='utf-8') as f:
                  f.write(encode_content)
                  f.close()
          except Exception as e:
              print(e)
  
      ''' 读取保存的html，通过pyquery读取到小说每一章的url，存入数据库,默认开始所有的状态都没none
      ID         title              url       status
      1460|第1461章 万龙巢穴（1）|2097128.html|none
      1461|第1462章 万龙巢穴（2）|2097129.html|none
      1462|第1463章 万龙巢穴（3）|2097130.html|none
      1463|第1464章 万龙巢穴（4）|2097131.html|none
      1464|第1465章 万龙巢穴（5）|2097132.html|none
      1465|第1466章 万龙巢穴（6）|2097133.html|none
      1466|第1467章 师徒大战（1）|2097134.html|none
      1467|第1468章 师徒大战（2）|2097135.html|none
      1468|第1469章 师徒大战（3）|2097136.html|none
      1469|第1470章 师徒大战（4）|2097137.html|none       
       '''  
      def save_book_list(self):
          with open("booklist.html","r",encoding="utf-8")as f:
              content = f.read()
              doc = pq(content)
              items=doc('.mulu_list li').items()
              status = {}
              conn = sqlite3.connect("xiaoshuo.db")
              cursor = conn.cursor()
              id = 0
              for each in items:
                  title_ = each.text()
                  url_ = each.find('a').attr('href')
                  cursor.execute("INSERT INTO booklist (id,title,url,status) VALUES ('{0}','{1}','{2}','none')".format(id,title_,url_))
                  id = id + 1
                  conn.commit()
              conn.close()
      ''' 
      下载小说，下载成功的章节会改变status为True，暂停之后下次下载会从第一个none的节点开始下载。
      sqlite> select * from booklist ;
      0|第1章 孔家逆子|2095666.html|True
      1|第2章 红光乍现|2095667.html|True
      2|第3章 炼精化气|2095668.html|True
      3|第4章 九阴神爪|2095669.html|True
      4|第5章 华家月儿|2095670.html|True
      5|第6章 五禽五行|2095671.html|none
      6|第7章 阶下之囚|2095672.html|none
      7|第8章 五行均衡|2095673.html|none
      8|第9章 月儿授艺|2095674.html|none
      9|第10章 周身穴窍|2095675.html|none 
      '''
      def down_book(self):
          conn = sqlite3.connect("xiaoshuo.db")
          cursor = conn.cursor()
          cursor.execute("select count(*) from booklist where status = 'none'")
          sum_num = cursor.fetchall()[0][0]
          cursor.execute("select * from booklist where status = 'none' limit 1")
          id  = int(cursor.fetchall()[0][0])
          for i in range(sum_num):
              time.sleep(3)
              cursor.execute("select * from booklist where ID = {0}".format(id))
              data = cursor.fetchall()
              try:
                  url = self.url + data[0][2]
                  req=requests.get(url=url,headers=header,timeout=20)
                  encodings = requests.utils.get_encodings_from_content(req.text)
                  encoding = req.apparent_encoding
                  encode_content = req.content.decode(encoding, 'replace')
                  doc=pq(encode_content)
                  item=doc('.contentbox').text()
                  with open(self.name,'a',encoding='utf-8') as f:
                      f.write(data[0][1]+'\n')                
                      f.write(item)
                      cursor.execute("UPDATE booklist SET status = ? where id = ?",("True","{0}".format(id)))
                      print("download sucess {0} {1}".format(data[0][1],url))
                      conn.commit()
                      id = id + 1
              except  Exception as e:
                  print(e)            
          conn.close()
  if __name__=='__main__':
      ''' 
      初始化，判断是否是第一次执行，失败后再执行一次继续下载。
      '''
      start = crawle(header,url,name)
      conn = sqlite3.connect("xiaoshuo.db")
      cursor = conn.cursor()
      cursor.execute("select count(*) from booklist where status = 'none'")
      sum_num = cursor.fetchall()[0][0]
      if sum_num:
          conn.close()
          start.down_book()      
      else:
          conn.close()
          start.down_book_list()
          start.save_book_list()
          start.down_book()
          conn.close()
  

•