python 版权保护,python爬虫篇4——爬取专利著作权信息

mysql代码：

CREATE TABLE `copyright` (

`id` int(11) NOT NULL AUTO_INCREMENT,

`软件名称` varchar(500) DEFAULT NULL,

`登记号` varchar(500) DEFAULT NULL,

`分类号` varchar(500) DEFAULT NULL,

`软件简称` varchar(500) DEFAULT NULL,

`版本号` varchar(500) DEFAULT NULL,

`首次发表日期` varchar(500) DEFAULT NULL,

`登记批准日期` varchar(500) DEFAULT NULL,

`软件著作权人` varchar(500) DEFAULT NULL,

`软件著作权人详情` varchar(500) DEFAULT NULL,

PRIMARY KEY (`id`),

UNIQUE KEY `登记号` (`登记号`)

) ENGINE=InnoDB AUTO_INCREMENT=9871 DEFAULT CHARSET=utf8

CREATE TABLE `patent` (

`id` int(11) NOT NULL AUTO_INCREMENT,

`专利名称` varchar(500) DEFAULT NULL,

`发明人` varchar(500) DEFAULT NULL,

`申请人` varchar(500) DEFAULT NULL,

`申请日` datetime DEFAULT NULL,

`公开日` datetime DEFAULT NULL,

`详情地址` varchar(500) DEFAULT NULL,

PRIMARY KEY (`id`),

UNIQUE KEY `详情地址` (`详情地址`)

) ENGINE=InnoDB AUTO_INCREMENT=13610 DEFAULT CHARSET=utf8

python代码：

getpatentdata.py主程序

import re

from urllib.parse import unquote, quote

from lxml import etree

from requests_html import HTMLSession

from 抓取专利著作权信息.MysqlHelper import MysqlHelper

# 获取专利信息

class Patent:

def __init__(self, sqr, year):

self.helper = MysqlHelper(host='localhost',

port=8080,

user='root',

passwd='123',

db='students',

charset='utf8')

self.creatTable()

self.sum = 0

while year <= 2019:

if year >= 2016:

dateList = ["%s-01-01" % str(year), "%s-02-01" % str(year), "%s-03-01" % str(year),

"%s-04-01" % str(year), "%s-05-01" % str(year), "%s-06-01" % str(year),

"%s-07-01" % str(year), "%s-08-01" % str(year), "%s-09-01" % str(year),

"%s-10-01" % str(year), "%s-11-01" % str(year), "%s-12-01" % str(year),

"%s-12-31" % str(year)]

else:

dateList = ["%s-01-01" % str(year), "%s-03-01" % str(year), "%s-05-01" % str(year),

"%s-07-01" % str(year),

"%s-09-01" % str(year),

"%s-11-01" % str(year), "%s-12-31" % str(year)]

print("*" * 66)

print("\033[36m开始抓取%s年的专利数据，已累计抓取%s条数据\033[0m" % (str(year), self.sum))

print("*" * 66)

for i in range(len(dateList) - 1):

self.getPatent(sqr, dateList[i], dateList[i + 1])

else:

year += 1

else:

print("\033[34m专利数据抓取完毕！共抓取%s条数据\033[0m" % str(self.sum))

showFunction()

# 建表

def creatTable(self):

sql1 = "CREATE TABLE `patent` (`id` int primary key not null auto_increment,`专利名称` varchar(500) DEFAULT NULL ,`发明人` varchar(500) DEFAULT NULL ,`申请人` varchar(500) DEFAULT NULL,`申请日` datetime DEFAULT NULL,`公开日` datetime DEFAULT NULL,`详情地址` varchar(500) DEFAULT NULL UNIQUE);"

self.helper.execute(sql1)

# 获取数据 sqr申请人 sqday_start申请日开始 sqday_end申请日结束

def getPatent(self, sqr, sqday_start, sqday_end):

self.patent_url = "http://dbpub.cnki.net/Grid2008/Dbpub/Brief.aspx?curpage=8&RecordsPerPage=350&QueryID=64&ID=SCPD&turnpage=1&systemno=&NaviDatabaseName=SCPD_ZJCLS&NaviField=%e4%b8%93%e9%a2%98%e5%ad%90%e6%a0%8f%e7%9b%ae%e4%bb%a3%e7%a0%81&navigatorValue=&subBase=all"

self.session = HTMLSession()

self.add_url = "http://dbpub.cnki.net/Grid2008/Dbpub/"

self.headers = {'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',

'Accept-Encoding': 'gzip,deflate',

'Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2',

'Connection': 'keep-alive',

'Content-Type': 'application/x-www-form-urlencoded',

'Origin': 'http://dbpub.cnki.net',

'Host': 'dbpub.cnki.net',

'Upgrade-Insecure-Requests': '1',

'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:70.0) Gecko/20100101 Firefox/70.0'

}

# h = self.session.post(self.patent_url, headers=headers)

# pagenum = etree.HTML(h.html.html).xpath('//div[@id="id_grid_total"]/text()')[0][5:-3]

# print("共%s条数据" % pagenum)

# self.patent_url = self.patent_url + "&RecordsPerPage=" + pagenum

self.data = "ID=SCPD&hdnSearchType=&