python爬虫实战笔记（7）selenium

功能：模拟人工动态操作浏览器
简单例程

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait

chrome_driver = r'C:\Users\lenovo\AppData\Local\Google\Chrome\Application\chromedriver.exe'
browser = webdriver.Chrome(executable_path=chrome_driver)
try:
    browser.get('http://www.baidu.com')
    input = browser.find_element_by_id('kw')
    input.send_keys('Python')
    input.send_keys(Keys.ENTER)
    wait = WebDriverWait(browser, 10)
    wait.until(EC.presence_of_element_located((By.ID, 'content_left')))
    print(browser.current_url)
    print(browser.get_cookie(()))
    print(browser.page_source)
finally:
    browser.close()

除chrome也同时支持firefox， edge， android等浏览器

声明浏览器对象

browser = webdriver.Chrome()
browser = webdriver.Firefox()
browser = webdriver.Edge()
browser = webdriver.PhantomJS()
browser = webdriver.Safari()

访问页面

browser.get('http://www.baidu.com')
print(browser.page_source)

pagesourse为源代码

查找结点

单个

browser.find_element_by_id()
browser.find_element_by_name()
browser.find_element_by_class_name()
browser.find_element_by_css_selector()
browser.find_element(By.ID, 'q')
等

by后面为查找方式

多个

find_elements_by+后缀（’d'）
find_elements(By.fangshi，‘d'）

节点交互

browser.get('https://www.taobao.com')
    input = browser.find_element_by_id('q')
    input.send_keys('iphone')
    time.sleep(1)
    input.clear()
    input.send_keys('ipad')
    button = browser.find_element_by_class_name('btn-search')
    button.click()

动态链操作

不针对特定节点的动作如鼠标拖曳，键盘输入

browser.get('http://www.runoob.com/try/try.php?filename=jqueryui-api-droppable')
    browser.switch_to.frame('iframeResult')
    sourse = browser.find_element_by_id('draggable')
    target = browser.find_element_by_id('droppable')
    actions = ActionChains(browser)
    actions.drag_and_drop(sourse,target)
    actions.perform()

其余见

执行javascript操作

有些操作selenium没有直接的功能函数，可以直接执行JavaScript的命令
如拖曳进度条

browser.get('http://www.zhihu.com/explore')
   browser.execute_script('window.scrollTo(0,document.body.scrollHeight)')
   browser.execute_script('alert("To Bottom")')

获取节点信息

logo = browser.find_element_by_id('zh-top-link-logo')
print(logo)
print(logo.get_attribute('class'))

print(imput.text)
print(imput.id)
print(imput.location)
print(imput.tag_name)
print(imput.size)

切换frame

url2 = 'http://www.runoob.com/try/try.php?filename=jqueryui-api-droppable'

browser.get(url2)
browser.switch_to.frame('iframeResult')
try:
    logo = browser.find_element_by_class_name('logo')
except NoSuchElementException:
    print('no logo')
browser.switch_to.parent_frame()
logo = browser.find_element_by_class_name('logo')
print(logo)
print(logo.text)

延时等待

p258

前进与后退

browser.get(url1)
browser.get(url2)
browser.get(url3)
browser.back()
time.sleep(1)
browser.forward()
browser.close()

cookies

browser.get(url1)
print(browser.get_cookies())
browser.add_cookie({'name': 'dd', 'domain': 'www.zhihu.com', 'value': 'germey'})
print(browser.get_cookies())
browser.delete_all_cookies()
print(browser.get_cookies())

选项卡管理

browser.get(url4)
browser.execute_script('window.open()')
print(browser.window_handles)
browser.switch_to.window(browser.window_handles[1])
browser.get(url3)
browser.switch_to.window(browser.window_handles[0])
browser.get(url2)

异常处理

browser.get(url4)
try:
    browser.get(url3)
except:
    print('time out')
try:
    browser.find_element_by_class_name('kk')
except NoSuchElementException:
    print('find no')
finally:
    browser.close()

无弹出页面模式

chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--headless')
browser = webdriver.Chrome(executable_path=chrome_driver, chrome_options=chrome_options)

对接FireFox

browser = webdriver.Firefox(...)