python按关键词爬取专利，并保存为EXCEL文件

#从专利网www.patent9.com爬取按关键词搜索的专利，并保存摘要至excel文件，方便对比分析

#需要chrome浏览器，也可按自己需求改写为其它浏览器

#运行python 3.9以上

# Author: trampler
#只能搜索200条

#插入库
import pandas as pd
import numpy as np

import openpyxl
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time
from bs4 import BeautifulSoup

import urllib.parse

#程序主体

keyword = '算法' #输入搜索关键词
wd = urllib.parse.quote(keyword.encode('utf-8'))
inputkw = urllib.parse.quote(wd.encode('gb2312'))

#要求有Chrome浏览器
driver = webdriver.Chrome()

driver.get('https://www.patent9.com')
wait = WebDriverWait(driver, 1)
searchtext=keyword
src=r'.\test1.xlsx'
dsc=r'.\dsc.xlsx'
x1=pd.ExcelFile(src)
algorithm=pd.read_excel(x1,x1.sheet_names[0],index_col=0,header=0,usecols=[0,1,2,3,4,5]) #Sheet5, 第一列索引，第一行索引，提取1，2，3，4，5，6列数据

time.sleep(1)

input_add = driver.find_element(By.XPATH,"//input[@name='k']") # 搜索框
input_add.clear()
input_add.send_keys(searchtext)
button = driver.find_element(By.XPATH,"//input[@type='submit']")
button.click()
source = driver.page_source
soup = BeautifulSoup(source, 'lxml')

total1 = str(soup.select("#form1 > span.direct.current > a")[0].text) # 获取申请发明总数
total2 = str(soup.select("#form1 > span:nth-child(6) > a")[0].text) # 获取申请实用新型总数
total3 = str(soup.select("#form1 > span:nth-child(7) > a")[0].text) # 获取申请外观专利总数

count1=0 
count2=10 #每页10个数据
all_list = []
all_list.append(["名称", "申请号", "申请人", "申请日", "摘要"])

idx = [id for id in range(1,200)]
df = pd.DataFrame(algorithm, index=idx)

with pd.ExcelWriter(dsc) as writer: #保存清洗后数据
    for i in range(1,2):    #下载页数
        url=u'https://www.patent9.com/search.aspx?k={inputkw}&c=PatentName&t=fmzl&sort=0&f={total1}&s={total2}&w={total3}&p={i}'    
        driver.get(url.format(inputkw=inputkw,total1=total1,total2=total2,total3=total3,i=i))
        wait = WebDriverWait(driver, 1)
        if i==math.ceil(int(total1)/10):
            count2=int(total1)-math.floor(int(total1)/10)*10
        for j in range(count2):
            j10=10+j*7 #每个数据相差7个div
            j11=j10+1
            j12=j10+2
            j14=j10+4
        
            patent_name = soup.select("#form1 > div:nth-child({}) > a span".format(j10))[0].text # 获取申请名称
            patent_no = soup.select("#form1 > div:nth-child({}) > a".format(j11))[0].text # 获取申请号
            patent_form = soup.select("#form1 > div:nth-child({}) > a".format(j12))[0].text # 获取申请人
            patent_abstract = soup.select("#form1 > div:nth-child({})".format(j14))[0].text # 获取申请摘要

            patent_date = soup.select("#form1 > div:nth-child({})".format(j12))[0].text
            patent_date=patent_date.replace('  ',':').split(':')[2]  # 获取申请日期
            time.sleep(0.5)
            df.loc[(i-1)*10+j,'名称']=patent_name
            df.loc[(i-1)*10+j,'申请号']=patent_no
            df.loc[(i-1)*10+j,'申请人']=patent_form
            df.loc[(i-1)*10+j,'申请日']=patent_date
            df.loc[(i-1)*10+j,'摘要']=patent_abstract    

     #数据保存 
    df.to_excel(writer)