python爬虫篇4——爬取专利著作权信息

mysql代码：

CREATE TABLE `copyright` (
  `id` int(11) NOT NULL AUTO_INCREMENT,
  `软件名称` varchar(500) DEFAULT NULL,
  `登记号` varchar(500) DEFAULT NULL,
  `分类号` varchar(500) DEFAULT NULL,
  `软件简称` varchar(500) DEFAULT NULL,
  `版本号` varchar(500) DEFAULT NULL,
  `首次发表日期` varchar(500) DEFAULT NULL,
  `登记批准日期` varchar(500) DEFAULT NULL,
  `软件著作权人` varchar(500) DEFAULT NULL,
  `软件著作权人详情` varchar(500) DEFAULT NULL,
  PRIMARY KEY (`id`),
  UNIQUE KEY `登记号` (`登记号`)
) ENGINE=InnoDB AUTO_INCREMENT=9871 DEFAULT CHARSET=utf8

CREATE TABLE `patent` (
  `id` int(11) NOT NULL AUTO_INCREMENT,
  `专利名称` varchar(500) DEFAULT NULL,
  `发明人` varchar(500) DEFAULT NULL,
  `申请人` varchar(500) DEFAULT NULL,
  `申请日` datetime DEFAULT NULL,
  `公开日` datetime DEFAULT NULL,
  `详情地址` varchar(500) DEFAULT NULL,
  PRIMARY KEY (`id`),
  UNIQUE KEY `详情地址` (`详情地址`)
) ENGINE=InnoDB AUTO_INCREMENT=13610 DEFAULT CHARSET=utf8

python代码：

getpatentdata.py主程序

import re
from urllib.parse import unquote, quote
from lxml import etree
from requests_html import HTMLSession

from 抓取专利著作权信息.MysqlHelper import MysqlHelper


# 获取专利信息
class Patent:
    def __init__(self, sqr, year):
        self.helper = MysqlHelper(host='localhost',
                                  port=8080,
                                  user='root',
                                  passwd='123',
                                  db='students',
                                  charset='utf8')
        self.creatTable()
        self.sum = 0
        while year <= 2019:
            if year >= 2016:
                dateList = ["%s-01-01" % str(year), "%s-02-01" % str(year), "%s-03-01" % str(year),
                            "%s-04-01" % str(year), "%s-05-01" % str(year), "%s-06-01" % str(year),
                            "%s-07-01" % str(year), "%s-08-01" % str(year), "%s-09-01" % str(year),
                            "%s-10-01" % str(year), "%s-11-01" % str(year), "%s-12-01" % str(year),
                            "%s-12-31" % str(year)]
            else:
                dateList = ["%s-01-01" % str(year), "%s-03-01" % str(year), "%s-05-01" % str(year),
                            "%s-07-01" % str(year),
                            "%s-09-01" % str(year),
                            "%s-11-01" % str(year), "%s-12-31" % str(year)]
            print("*" * 66)
            print("\033[36m开始抓取%s年的专利数据，已累计抓取%s条数据\033[0m" % (str(year), self.sum))
            print("*" * 66)
            for i in range(len(dateList) - 1):
                self.getPatent(sqr, dateList[i], dateList[i + 1])
            else:
                year += 1
        else:
            print("\033[34m专利数据抓取完毕！共抓取%s条数据\033[0m" % str(self.sum))
            showFunction()

    # 建表
    def creatTable(self):
        sql1 = "CREATE TABLE `patent` (`id` int primary key not null auto_increment,`专利名称` varchar(500) DEFAULT NULL  ,`发明人` varchar(500) DEFAULT NULL ,`申请人` varchar(500) DEFAULT NULL,`申请日` datetime DEFAULT NULL,`公开日` datetime DEFAULT NULL,`详情地址` varchar(500) DEFAULT NULL UNIQUE);"
        self.helper.execute(sql1)

    # 获取数据 sqr申请人 sqday_start申请日开始 sqday_end申请日结束
    def getPatent(self, sqr, sqday_start, sqday_end):
        self.patent_url = "http://dbpub.cnki.net/Grid2008/Dbpub/Brief.aspx?curpage=8&RecordsPerPage=350&QueryID=64&ID=SCPD&turnpage=1&systemno=&NaviDatabaseName=SCPD_ZJCLS&NaviField=%e4%b8%93%e9%a2%98%e5%ad%90%e6%a0%8f%e7%9b%ae%e4%bb%a3%e7%a0%81&navigatorValue=&subBase=all"
        self.session = HTMLSession()
        self.add_url = "http://dbpub.cnki.net/Grid2008/Dbpub/"
        self.headers = {'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
                        'Accept-Encoding': 'gzip,deflate',
                        'Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2',
                        'Connection': 'keep-alive',
                        'Content-Type': 'application/x-www-form-urlencoded',
                        'Origin': 'http://dbpub.cnki.net',
                        'Host': 'dbpub.cnki.net',
                        'Upgrade-Insecure-Requests': '1',
                        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:70.0) Gecko/20100101 Firefox/70.0'
                        }
        # h = self.session.post(self.patent_url, headers=headers)
        # pagenum = etree.HTML(h.html.html).xpath('//div[@id="id_grid_total"]/text()')[0][5:-3]
        # print("共%s条数据" % pagenum)
        # self.patent_url = self.patent_url + "&RecordsPerPage=" + pagenum
        self.data = "ID=SCPD&hdnSearchType=&hdnIsAll&#