利用python+selenium爬取derwent数据库上的patents

需求：
登陆web of science，并进入derwent数据库，按照公司excel列表依次进行搜索，并将所有搜索道德记录下载至本地指定路径。

思路：和爬取cortellis的步骤基本相近，也是利用selenium模拟浏览器，主要涉及的模拟有：下拉菜单、按钮、input框，总体上讲并不复杂。
然而需要注意的是，derwent搜索结果如果为0，它会返回一个错误页面，当出现这种情况可以用一个try-except抓出来，再用selenium执行一次刷新即可。

缺点是大量的使用了try-except，也没有太好的解决方法，这个数据库的随机性实在是太大。

下面是代码：

from selenium import webdriver
import sys,io
from selenium.webdriver.common.by import By
import time
import xlrd
import json
import re,os
import requests
from selenium.webdriver.support.ui import Select
from openpyxl import Workbook
import pickle


def read_company(fileName):
    bk=xlrd.open_workbook(fileName)
    shxrange=range(bk.nsheets)
    try:
        sh=bk.sheet_by_name("Sheet1")
    except:
        print ("代码出错")
    book = Workbook(encoding='utf-8')
    UPC = []
    tmp1 = sh.col_values(0)[1:]  #company
    return tmp1
#获取当前页面的cookie
def get_current_cookie():
	cookie = browser.get_cookies()
	jsonCookies = json.dumps(cookie)
	#拼接
	cookie_current = [item['name'] + '=' + item['value'] for item in cookie]
	#这里注意每一个字段之间要有"; "分隔，否则无效
	cookiestr = '; '.join(item for item in cookie_current)
	return cookiestr

#设置下载路径
def set_chrome_pref(path):
    chromeOptions = webdriver.ChromeOptions()
    prefs = {"download.default_directory": path}
    chromeOptions.add_experimental_option("prefs", prefs)
    time.sleep(1)
    return chromeOptions

# from openpyxl import Workbook as wb

#防止出现命令行无法识别
sys.stdout = io.TextIOWrapper(sys.stdout.buffer,encoding='gb18030')
fileName0 = './20190710_Zhangbo_Firm list.xlsx'

print('load company name...')
company_list = read_company(fileName0)
flag = 0 #第一次运行
for company in company_list:
	print('search company patent for: ' + company)
	download_path = "G:\\spider_study\\selenium\\derwent_result\\" + company + '\\'
	isExists = os.path.exists(download_path) 
	if not isExists:
		os.makedirs(download_path)
	tmp = set_chrome_pref(download_path)
	#定义浏览器载体
	browser = webdriver.Chrome(chrome_options=tmp)
	browser.get("http://apps.webofknowledge.com/DIIDW_GeneralSearch_input.do?product=DIIDW&search_mode=GeneralSearch&SID=7FGhAhNWvHXPFIknaix&preferencesSaved=")

	#下拉菜单选择derwent数据库
	switch_database = browser.find_element_by_class_name('select2')
	switch_database = switch_database.find_element_by_class_name('select2-selection')
	switch_database.click()

	derwent_option = browser.find_elements_by_class_name('select2-results__option')[4]
	derwent_option.click()

	#选择授权人
	assignee_option = browser.find_element_by_id('select2-select1-container')
	assignee_option.click()
	assignee_option = browser.find_elements_by_class_name('select2-results__option')[3]
	assignee_option.click()

	# #定位serach模块
	search_block = browser.find_element_by_class_name('block-search-content')
	search_block = search_block.find_element_by_class_name('focusinput')
	time.sleep(1)
	search_block.send_keys(company)
	# #
	search_button = browser.find_elements_by_class_name('large-button')[0]
	search_button.click()

	#点击按钮后必须switch_to才能定位成功新的窗口
	browser.switch_to.window(browser.window_handles[0])
	time.sleep(10)
	try:
		items_per_page = browser.find_element_by_id('select2-selectPageSize_bottom-container')
		items_per_page.click()
		items_50_per_page = browser.find_elements_by_class_name('select2-results__option')[-1]
		items_50_per_page.click()
		time.sleep(10)
	except:
		browser.quit()
		continue
		
	#获得总页数
	try:
		page_num = browser.find_element_by_id('pageCount.top').text
		print('There are ' + str(page_num) + ' pages for this company!')
	except:
		browser.refresh();



	for i in range(int(page_num)):

		#全选当前页面所有专利
		try:
			select_all = browser.find_element_by_id('SelectPageChkId')
			select_all.click()		
		except:
			browser.navigate().refresh();


		if flag == 0: #如果是第一次打开浏览器
			#选择导出格式
			export_type = browser.find_elements_by_id('exportTypeName')[0]
			export_type.click()
			other_file = browser.find_elements_by_class_name('quickOutputOther')[0]
			other_file.click()


			#下拉菜单选择格式
			all_record = browser.find_element_by_id('numberOfRecordsAllOnPage')
			all_record.click()

			record_content = browser.find_element_by_id('select2-bib_fields-container')
			record_content.click()
			full_record = browser.find_elements_by_class_name('select2-results__option')[-1]
			full_record.click()



			file_type = browser.find_element_by_id('select2-saveOptions-container')
			file_type.click()
			tab_file = browser.find_elements_by_class_name('select2-results__option')[3]
			tab_file.click()
			flag = 1

		else: #如果已经保存了输出格式（不是第一次打开）
			#点击export按钮
			export_block = browser.find_elements_by_class_name('onload-secondary-button')[0]
			export_block.click()
			time.sleep(1)

		#导出下载
		export_button = browser.find_elements_by_class_name('onload-primary-button')[-1]
		export_button.click()
		time.sleep(3)


		#关闭下载页面
		close_button = browser.find_element_by_class_name('flat-button')
		close_button.click()
		time.sleep(1)

		#点击下一页
		nextpage_button = browser.find_elements_by_class_name('paginationNext')[0]
		nextpage_button.click()
		browser.switch_to.window(browser.window_handles[0])
		time.sleep(8)
	browser.quit()
	flag = 0 #初始化