import os
from browsermobproxy import Server
from selenium import webdriver
import time
# 初始化代理
def init_proxy():
server =Server(os.path.join(os.getcwd(), r'browsermob-proxy-2.1.4\bin\browsermob-proxy.bat'))
server.start()return server.create_proxy()
# 初始化浏览器,传递代理对象
def init_browser(proxy):
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--headless') # 无头浏览器
chrome_options.add_argument('ignore-certificate-errors') # 无视证书验证
chrome_options.add_argument('--start-maximized') # 开始时直接最大屏幕
# 设置用户数据目录,避免每次都要重新登陆
chrome_options.add_argument(r'--user-data-dir=D:\ChromeUserData')
chrome_options.add_argument('--proxy-server={0}'.format(proxy.proxy)) # 设置请求的代理
return webdriver.Chrome(options=chrome_options)if __name__ =='__main__':
server_proxy =init_proxy()
browser =init_browser(server_proxy)
# 监听结果
server_proxy.new_har("test", options={'captureContent': True,'captureHeaders': True
})
browser.get("https://www.baidu.com")
time.sleep(2)
catch_result = server_proxy.har
for entry in catch_result['log']['entries']:print(entry['response']['content'])
# 用完记得关
server_proxy.close()
browser.close()
下面是封装使用
# _*_ coding:utf-8 _*_
import os
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from browsermobproxy import Server
import time
import json
class BaseFramework(object):
def __init__(self):
# 修改下载的文件路径
self.server =Server(r'F:\BrowserMobProxy\browsermob-proxy-2.1.4\bin\browser-mob-proxy.bat')
self.server.start()
self.proxy = self.server.create_proxy()
chrome_options =Options()
chrome_options.add_argument('--ignore-certificate-errors')
chrome_options.add_argument('--proxy-server={0}'.format(self.proxy.proxy))
chrome_options.add_argument('--headless') # 无头模式
self.browser = webdriver.Chrome(options=chrome_options)
def process_request(self, request, response):
pass
def process_response(self, response, request):
pass
def run(self, func,*args):
self.proxy.new_har(options={'captureContent': True,'captureHeaders': True
})func(*args)
result = self.proxy.har
for entry in result['log']['entries']:
request = entry['request']
response = entry['response']
self.process_request(request, response)
self.process_response(response, request)
def __del__(self):
self.proxy.close()
self.browser.close()
class Framework(BaseFramework):
def load(self, url):
self.browser.get(url)
time.sleep(3)
def process_request(self, request, response):
pass
def process_response(self, response, request):#print(request['url'])
# 找到你所需数据的url即可快乐的解析数据了
if'/item/timemap/cn/' in request['url']:
try:
text = response['content']['text']
text_dict = json.loads(text)
data_result = text_dict['data']
except KeyError:print('----KeyError: text----')return
name = data_result['name'] # 姓名
id_name = name_id +'_'+ name
print(id_name)
time_map_list = data_result['timeMap']if time_map_list:
time_map_dict ={}for i in range(len(time_map_list)):
time_map = time_map_list[i]
time_map_dict[str(i)]= time_map
else:return
path = f'./****/{id_name}.json'if os.path.exists(path):print(f'------{id_name}--已存在------')return
with open(path,'w', encoding='utf-8') as f:
f.write(json.dumps(time_map_dict, ensure_ascii=False, indent=4))if __name__ =='__main__':
Framework =Framework()
id_list =['********']for name_id in id_list:
url ="************************"
Framework.run(Framework.load, url)
解释解释:
代码一共分了四步:
•第一步便是启动 BrowserMob Proxy,它会在本地启动一个代理服务,这里注意 Server 的第一个参数需要指定 BrowserMob Proxy 的可执行文件路径,这里我就指定了下载下来的 BrowserMob Proxy 的 bin 目录的 browsermob-proxy 的路径。
•第二步便是启动 Selenium 了,它可以设置 Proxy Server 为 BrowserMob Proxy 的地址。
•第三步便是访问页面同时监听结果,这里我们需要调用 new_har 方法,同时指定捕获 Resopnse Body 和 Headers 信息,紧接着调用 Selenium 的 get 方法访问一个页面,这时候浏览器便会加载这个页面,同时所有的请求和响应信息都会被记录到 HAR 中。
•第四步便是读取 HAR 到内容了,我们调用 log 到 entries 字段,里面便包含了请求和响应的具体结果,这样所有的请求和响应信息我们便能获取到了,Ajax 的内容也不在话下。
举个栗子1
from selenium import webdriver
from time import sleep
from selenium.webdriver.common.keys import Keys
import datetime
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.support.select import Select
from selenium.webdriver.chrome.options import Options
def qinjia(browser,url):
browser.get(url)sleep(1)
browser.implicitly_wait(3)WebDriverWait(browser,5).until(EC.presence_of_all_elements_located((By.ID,"user_main")))
user_main_div=browser.find_element_by_id("user_main")
username_input=user_main_div.find_element_by_id("txtId") #用户名
password_input=user_main_div.find_element_by_id("txtMM") #密码
login_btn=user_main_div.find_element_by_id("IbtnEnter") #登录按钮
# 修改1:此处的账号和密码
username_input.send_keys("==================账号===================")
password_input.send_keys("==================密码===================")
login_btn.click()sleep(1)
browser.implicitly_wait(3)WebDriverWait(browser,5).until(EC.presence_of_all_elements_located((By.CLASS_NAME,"tabThinM")))
table_tag=browser.find_element_by_class_name("tabThinM")
href_body=table_tag.find_elements_by_tag_name("tbody")[2]
href=href_body.find_elements_by_tag_name("tr")[1].find_element_by_tag_name("a").get_attribute("href")
browser.get(href)
table_wjTA=browser.find_element_by_id("wjTA")
div_gerenjiankang=table_wjTA.find_elements_by_class_name("dvO")[0] #个人健康
div_shenqing=table_wjTA.find_elements_by_class_name("dvO")[1] #申请进入
# 个人健康
selects_tag=div_gerenjiankang.find_elements_by_tag_name("select")
work_station_select=selects_tag[2]
health_station_select=selects_tag[3]
live_station_select=selects_tag[4]
family_station_select=selects_tag[5]Select(work_station_select).select_by_value("1")Select(health_station_select).select_by_value("1")Select(live_station_select).select_by_value("1")Select(family_station_select).select_by_value("1")
#申请进入
select_shenqin_time_tags=div_shenqing.find_elements_by_tag_name("select")
input_shenqin_reaseons_tags=div_shenqing.find_elements_by_tag_name("input")
target_place_input=input_shenqin_reaseons_tags[0]
reason_input=input_shenqin_reaseons_tags[1]
# 修改2:成此处的申请目的地和事由
target_place_input.send_keys("=====================申请目的========================")
reason_input.send_keys("==========================事由===============================")Select(select_shenqin_time_tags[0]).select_by_value("1")Select(select_shenqin_time_tags[1]).select_by_value("06")Select(select_shenqin_time_tags[2]).select_by_value("3")Select(select_shenqin_time_tags[3]).select_by_value("23")
submit_input=browser.find_element_by_tag_name("input")
submit_input.click()
def log(message):
curent_time = datetime.datetime.now()print(curent_time)
f =open("log.txt","a+", encoding="utf-8")
f.write(str(curent_time)+": "+message+"\n")
f.close()
def headLessChrome():
chrome_driver = r"chromedriver.exe"
chrome_options=Options()
chrome_options.add_argument("--headless")
chrome_options.add_argument("--disable-gpu")
browser=webdriver.Chrome(options=chrome_options, executable_path=chrome_driver)return browser
if __name__ =='__main__':
url="http://login.cuit.edu.cn/Login/xLogin/Login.asp"
browser=headLessChrome()
try:qinjia(browser,url)log("成功")
except:log("失败")
browser.quit()
举个栗子2
from bs4 import BeautifulSoup
import requests
import time
import pandas as pd
from selenium import webdriver
import csv
import re
import numpy as np
import os
afterLogin_headers ={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36'}#driver= webdriver.PhantomJS(executable_path=r'D:\code\patent_info\phantomjs-2.1.1-windows\bin\phantomjs.exe', service_args=['--ignore-ssl-errors=true','--ssl-protocol=TLSv1'])
chrome_driver = r'D:\code\patent_info\chromedriver.exe'
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument(r'--user-data-dir=D:\code\patent_info\ChromeUserqichacha0623')
driver = webdriver.Chrome(executable_path = chrome_driver, options=chrome_options)#driver.maximize_window()
# # driver.add_argument('--headless') # 开启无界面模式
def get_company_message(company):'''
Input: company is a chinese word
Todo: get company code and patent url
Return: patent url
'''
driver.get('https://www.qcc.com/search?key={}'.format(company))
time.sleep(5)print('https://www.qcc.com/search?key={}'.format(company))
html_page = driver.page_source
soup =BeautifulSoup(html_page,features="lxml")
href = soup.find_all('a',{'class':'title'})[0].get('href')print('href is {}'.format(href))
href2 = href.replace('firm','cassets')print('href2 is {}'.format(href2))return href2
def iselementExist(element):'''
Input: xpath of patent_table element
Todo: Whether the patent exists
Return: bools
'''
flag = True
try:
driver.find_element_by_xpath(element)return flag
except:
flag=False
return flag
# //*[@id="zhuanlilist"]/div[1]/h3
def get_patent_infomation(number_page):# get information single web
'''
Input: href2 get from 'get_company_message'
Todo: get patents information from table
Return: soup.select
'''
number_page =BeautifulSoup(number_page,features="lxml")
data_infos = number_page.select('#zhuanlilist .app-ntable td')return data_infos
def save_patient(data_infos, key_company):'''
Input:1.data_infos: get one page information from 'get_patent_infomation'2.key_company: company name read from excel
Todo: explain and write input to csv
return: None
'''
company_patient =[]for info in data_infos:
company_patient.append(info.text)
company_patient_classified =[company_patient[i:i+10]for i in range(0,len(company_patient),10)]
company_patient_classified = np.array(company_patient_classified) # 使用numpy中的array,将列表转化为标准的数组s
dataframe = pd.DataFrame({'company': key_company,'Number': company_patient_classified[:,0],'Patent_name': company_patient_classified[:,1],'Patent_type': company_patient_classified[:,2],'Patent_statu': company_patient_classified[:,3],'Application number': company_patient_classified[:,4],'Apply_data': company_patient_classified[:,5],'Public_announcement_No': company_patient_classified[:,6],'Public_announcement_Data': company_patient_classified[:,7],'Inventor': company_patient_classified[:,8],'More': company_patient_classified[:,9]})if not os.path.exists('D:\code\patent_info\data\output_data\company_patient1.csv'):
dataframe.to_csv("D:\code\patent_info\data\output_data\company_patient1.csv", index=False, sep=',', mode='a',encoding='gb18030')else:
dataframe.to_csv("D:\code\patent_info\data\output_data\company_patient1.csv", index=False, sep=',', mode='a',encoding='gb18030',header=False)
def whether_turn_page(element1):'''
Input: xpath of patent_table element
Todo: Whether the pages_number table exists
Return: bools
'''
flag1 = True
try:
driver.find_element_by_xpath(element1)return flag1
except:
flag1=False
return flag1
# //*[@id="zhuanlilist"]/div[4]/nav/ul
def turn_next_page(key_company):'''
Input: key_company: company name read from excel
Todo: design how to turn next page in defferent condition
return: key_company: company name read from excel
'''
print(driver.find_element_by_xpath('//*[@id="zhuanlilist"]/div[4]/nav/ul/li[last()]').text)
list_max = driver.find_element_by_xpath('//*[@id="zhuanlilist"]/div[4]/nav/ul/li[last()]').text
#num_max =int(re.sub("\D","", list_max)) # just keep int numberif list_max =='>': # patent number between(10,70]
list_second_max = driver.find_element_by_xpath('//*[@id="zhuanlilist"]/div[4]/nav/ul/li[last()-1]/a').text
for turn_index in range(int(list_second_max)-1):
driver.find_element_by_xpath('//*[@id="zhuanlilist"]/div[4]/nav/ul/li[last()]').click()
time.sleep(1)
number_page = driver.page_source
data_infos =get_patent_infomation(number_page)save_patient(data_infos, key_company)
elif int(re.sub("\D","", list_max)):#elifisinstance(num_max,int):# patent number above 70for turn_index in range(int(re.sub("\D","", list_max))-1):
driver.find_element_by_xpath('//*[@id="zhuanlilist"]/div[4]/nav/ul/li[last()-1]/a').click()
time.sleep(1)
number_page = driver.page_source
data_infos =get_patent_infomation(number_page)save_patient(data_infos, key_company)else:print('error company name is {}'.format(key_company))#next_page_button //*[@id="zhuanlilist"]/div[4]/nav/ul/li[8]/a
if __name__ == '__main__':
csv_file = r"D:\code\patent_info\patient1.csv"
with open(csv_file, encoding='utf-8') as csvfile:
reader=csv.reader(csvfile)
for i,key_company in enumerate(reader):
print('i is {}'.format(i))
print('rows is {}'.format(key_company))
# print(type(key_company))
key_company = ' '.join(key_company)
patent_url = get_company_message(key_company) # patent url
driver.get(patent_url)
time.sleep(1)
if iselementExist('//*[@id="zhuanlilist"]/div[1]/h3'):
number_page = driver.page_source
data_infos = get_patent_infomation(number_page)
save_patient(data_infos, key_company)
if whether_turn_page('//*[@id="zhuanlilist"]/div[4]/nav/ul'):
turn_next_page(key_company)