这两种自动化控制浏览器的模块我都用过,因为我用的是python操作,所以用的是pyppeteer库。
首先,对于基础操作来说,比如输密码登录,点击各种元素之类,selenium和puppeteer都能满足
环境上来说,selenium能支持更多的浏览器,puppeteer貌似只支持chrome?
部署上来说,selenium需要自己下载目前chrome浏览器对应版本的chromedriver,puppteer可以自己下载chrome开发版本,这个是puppteer稍微方便一点。
如果你是想处理某些坑爹的case,比如淘宝滑块验证码这种,貌似只能用puppteer。
语法上来看,selenium主要是同步语法,puppteer则是一堆异步,如果你是python或者java开发,那用selenium好太多了,如果是js开发可以考虑puppteer。
最后,selenium真香。
附两种代码例子
1 我封装的功能比较完整的selenium操作类
import os
from selenium import webdriver
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.common.keys import Keys
# 文档https://www.cnblogs.com/taceywong/p/6602927.html?utm_source=tuicool&utm_medium=referral
# 中文apihttp://selenium-python-zh.readthedocs.io/en/latest/
class ChromeOperate():
def __init__(self, url='', executable_path='', User_data_dir='', arguments=[], headless=False, no_pic=False):
option = webdriver.ChromeOptions()
if User_data_dir:
option.add_argument('--user-data-dir=%s' % User_data_dir) # 设置成用户自己的数据目录
else:
import getpass
username = getpass.getuser()
default_path = r'C:\Users\%s\AppData\Local\Google\Chrome\User Data' % username # echo %LOCALAPPDATA%\Google\Chrome\User Data
print(default_path)
if os.path.exists(default_path):
# option.add_argument('--user-data-dir=%s' % default_path)
pass
if no_pic:
prefs = {"profile.managed_default_content_settings.images": 2} # 如果出现图片显示不了需要在配置文件里改回1
option.add_experimental_option("prefs", prefs)
option.add_argument('--start-maximized')
if headless:
option.add_argument('headless')
option.add_argument('google-base-url=%s' % 'https://www.baidu.com/')
for argument in arguments:
option.add_argument(argument)
if not executable_path: executable_path = r'C:\Users\Administrator\Desktop\chromedriver.exe'
self.driver = webdriver.Chrome(executable_path=executable_path, chrome_options=option)
if url: self.open(url)
def open(self, url):
self.driver.get(url) # self.driver.get(url).page_source
def open_source(self):
return self.driver.page_source
def title(self):
self.title = self.driver.title
return self.title
def quit(self):
self.driver.quit()
def find_element_by_name(self, name):
return self.driver.find_element_by_name(name)
def find_elements_by_xpath(self, xpath): # 貌似不能用/text
return self.driver.find_elements_by_xpath(xpath)
def find_element_by_id(self, id):
try:
return self.driver.find_element_by_id(id)
except:
return None
def input_words(self, element, words):
element.clear()
element.send_keys(words)
def click_by_id(self, id):
self.driver.find_element_by_id(id).click()
def send_file(self, element, path):
element.sendKeys(path);
def wait_element(self, element_id):
WebDriverWait(self.driver, 10).until(
EC.presence_of_element_located((By.ID, element_id))
)
def get_title(self):
print(self.driver.title)
return self.driver.title
def refresh(self):
self.driver.refresh() #
def down_page(self):
from selenium.webdriver.common.keys import Keys
self.driver.execute_script("window.scrollBy(0,3000)")
def get_cookie(self):
cookies_list = self.driver.get_cookies()
cookie_dict = {i["name"]: i["value"] for i in cookies_list}
return cookie_dict
def close_tab(self):
ActionChains(self.driver).key_down(Keys.CONTROL).send_keys("w").key_up(Keys.CONTROL).perform()
def close_other_tab(self):
handle = self.driver.current_window_handle
# 获取当前所有窗口句柄(窗口A、B)
handles = self.driver.window_handles
# 对窗口进行遍历
for newhandle in handles:
# 筛选新打开的窗口B
if newhandle != handle:
# 切换到新打开的窗口B
# browser.switch_to_window(newhandle) 旧版本
self.driver.switch_to.window(newhandle)
# # 在新打开的窗口B中操作
# self.driver.find_element_by_id('xx').click()
# 关闭当前窗口B
self.driver.close()
# 切换回窗口A
self.driver.switch_to_window(handles[0])
'''
验证码截取
链式操作ActionChains(driver).move_to_element(menu).click(hidden_submenu).perform()
// Copy the element screenshot to disk
File screenshotLocation = new File("C:\\images\\GoogleLogo_screenshot.png");
FileUtils.copyFile(screenshot, screenshotLocation);
'''
if __name__ == '__main__':
cop = ChromeOperate(executable_path=r'chromedriver.exe')
cop.open("https://videoquickedit.com/")
print(cop.driver.current_url)
2 pyppeteer
import asyncio
import time
from pyppeteer.launcher import launch
class test(object):
def get_test(self):
loop = asyncio.new_event_loop()
self.loop = loop
asyncio.set_event_loop(loop)
loop.run_until_complete(self.main())
# self.loop.close()
async def main(self):
# 使用前需要确保关闭
url = 'https://www.baidu.com/s?wd=coroutine%20%27Launcher.killChrome%27%20was%20never%20awaited&rsv_spt=1&rsv_iqid=0xaa093ece0008eafe&issp=1&f=8&rsv_bp=1&rsv_idx=2&ie=utf-8&rqlang=cn&tn=baiduhome_pg&rsv_enter=1&rsv_t=2122Edd7Utwtaq3%2BC1k8RAJXlzih14qru1Nksz%2BhJn%2F1QqRgEI9GjSCqkLCfGY2YpbPq&oq=puppeteer%2520%25E6%25B8%2585%25E7%25A9%25BA%25E8%25BE%2593%25E5%2585%25A5&inputT=312&rsv_pq=f45dc490000b0918&rsv_sug3=31&rsv_sug1=18&rsv_sug7=100&rsv_n=2&rsv_sug2=0&rsv_sug4=745&rsv_sug=1'
browser = await launch({'headless': True, 'args': ['--no-sandbox'],'dumpio':True}) #executablePath 服务器需要设置 否则会下载个默认的headless': True,
self.browser = browser
page = await browser.newPage()
self.page = page
await page.setUserAgent(
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 Edge/16.16299')
await page.goto(url)
await page.type('#kw','啦啦')
await page.waitFor(2000)
for i in range(20):
await page.keyboard.press('ArrowRight');
await page.keyboard.press('Backspace');
pass
print(await page.content())
if __name__ == '__main__':
test_obj = test()
test_obj.get_test()
实际上对于更复杂的拦截我还有一种方案,就是自己写一个chrome插件,和后端脚本(我用的是python)用socket通信。我称之为js版本的webdriver。。这个插件我已经写好了,不过因为没有实际应用到所以就放着了,如果有人有兴趣的可以一起讨论下。