python3 2018分布式爬虫教程 -8 selenium库详解

selenium：自动化测试工具，在爬虫中主要解决javascript渲染的问题

安装命令：

pip install selenium

1.基本使用：

browser = webdriver.Chrome():声明浏览器驱动对象

input = browser.find_element_by_id('kw')：获取 id 为 kw 的标签（此处为输入框标签）

input.send_keys('Python')：将 'Python' 赋值给输入框

input.send_keys(Keys.ENTER)：回车

wait = WebDriverWait(browser, 10)

wait.until(EC.presence_of_element_located((By.ID, 'content_left')))：等待直到 id 为'content_left' 被加载出来

browser.current_url：当前url地址

browser.get_cookies()：当前cookies

browser.page_source：当前源码

# coding=utf-8

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait
from bs4 import BeautifulSoup

browser = webdriver.Chrome()
try:
    browser.get('https://www.baidu.com')
    input = browser.find_element_by_id('kw')
    input.send_keys('Python')
    input.send_keys(Keys.ENTER)
    wait = WebDriverWait(browser, 10)
    wait.until(EC.presence_of_element_located((By.ID, 'content_left')))
    print(browser.current_url)
    print("-----------------------------------------")
    print(browser.get_cookies())
    print("-----------------------------------------")
    html = browser.page_source
    soup = BeautifulSoup(html, 'lxml')
    print(soup.title.string)
finally:
    browser.close()

'''
https://www.baidu.com/s?ie=utf-8&f=8&rsv_bp=0&rsv_idx=1&tn=baidu&wd=Python&rsv_pq=c9f913b10000f7dd&rsv_t=ad72liBGsRpdg2QJkDXBGXIHiSlXo8lBSo6yBEvSGwqiOH8s4L%2FFQ7REdVg&rqlang=cn&rsv_enter=1&rsv_sug3=6&rsv_sug2=0&inputT=138&rsv_sug4=139
-----------------------------------------
[{'domain': '.baidu.com', 'httpOnly': False, 'name': 'H_PS_PSSID', 'path': '/', 'secure': False, 'value': '1469_21117_18559_28206_28131_28267_28139_22073'}, {'domain': '.baidu.com', 'httpOnly': False, 'name': 'delPer', 'path': '/', 'secure': False, 'value': '0'}, {'domain': '.baidu.com', 'expiry': 3694356829.484673, 'httpOnly': False, 'name': 'BAIDUID', 'path': '/', 'secure': False, 'value': 'BC1DFBAA5F91AF3DBF82E5DE55455B29:FG=1'}, {'domain': '.baidu.com', 'expiry': 3694356829.484719, 'httpOnly': False, 'name': 'BIDUPSID', 'path': '/', 'secure': False, 'value': 'BC1DFBAA5F91AF3DBF82E5DE55455B29'}, {'domain': '.baidu.com', 'expiry': 3694356829.48475, 'httpOnly': False, 'name': 'PSTM', 'path': '/', 'secure': False, 'value': '1546873184'}, {'domain': 'www.baidu.com', 'httpOnly': False, 'name': 'BD_HOME', 'path': '/', 'secure': False, 'value': '0'}, {'domain': '.baidu.com', 'expiry': 1546959583.039086, 'httpOnly': False, 'name': 'BDORZ', 'path': '/', 'secure': False, 'value': 'B490B5EBF6F3CD402E515D22BCDA1598'}, {'domain': 'www.baidu.com', 'expiry': 1547737182, 'httpOnly': False, 'name': 'BD_UPN', 'path': '/', 'secure': False, 'value': '12314553'}, {'domain': 'www.baidu.com', 'httpOnly': False, 'name': 'BD_CK_SAM', 'path': '/', 'secure': False, 'value': '1'}, {'domain': '.baidu.com', 'httpOnly': False, 'name': 'PSINO', 'path': '/', 'secure': False, 'value': '3'}, {'domain': 'www.baidu.com', 'expiry': 1546875776, 'httpOnly': False, 'name': 'H_PS_645EC', 'path': '/', 'secure': False, 'value': '87ae%2BXfV7LZ4voylGO%2BdlHBgXW6mRW9%2Fng6yiZ8RFMb3BmqYntOJizpi5Yg'}]
-----------------------------------------
Python_百度搜索
'''

2.声明浏览器对象：

from selenium import webdriver

browser = webdriver.Chrome()
browser = webdriver.Firefox()
browser = webdriver.Edge()
browser = webdriver.PhantomJS()
browser = webdriver.Safari()

3.访问页面，获取源码：

from selenium import webdriver

browser = webdriver.Chrome()
browser.get('https://www.taobao.com')
print(browser.page_source)
browser.close()

4.查找元素

（1）查找单个元素：

• find_element_by_name：通过 name 查找
• find_element_by_xpath：通过 xpath 查找
• find_element_by_link_text：通过 link 查找
• find_element_by_partial_link_text
• find_element_by_tag_name：通过 tag 查找
• find_element_by_class_name：通过 class查找
• find_element_by_css_selector：通过 css选择器 查找

# coding=utf-8

from selenium import webdriver

browser = webdriver.Chrome()
browser.get('https://www.taobao.com')
input_first = browser.find_element_by_id('q')
input_second = browser.find_element_by_css_selector('#q')
input_third = browser.find_element_by_xpath('//*[@id="q"]')
print(input_first)
print(input_second)
print(input_third)
browser.close()

'''
<selenium.webdriver.remote.webelement.WebElement (session="0b2dfb2d032cb9dc78894fa080be6c38", element="0.7417393687379614-1")>
<selenium.webdriver.remote.webelement.WebElement (session="0b2dfb2d032cb9dc78894fa080be6c38", element="0.7417393687379614-1")>
<selenium.webdriver.remote.webelement.WebElement (session="0b2dfb2d032cb9dc78894fa080be6c38", element="0.7417393687379614-1")>
'''

from selenium import webdriver
from selenium.webdriver.common.by import By

browser = webdriver.Chrome()
browser.get('https://www.taobao.com')
input_first = browser.find_element(By.ID, 'q')
print(input_first)
browser.close()


'''
<selenium.webdriver.remote.webelement.WebElement (session="6aee910717374b487b406d92e6114b1c", element="0.4977152944369816-1")>
'''

（2）查找多个元素：

• find_elements_by_name
• find_elements_by_xpath
• find_elements_by_link_text
• find_elements_by_partial_link_text
• find_elements_by_tag_name
• find_elements_by_class_name
• find_elements_by_css_selector

find_elements_by_css_selector：通过css选择器查找 class为service-bd里面的所有 li 标签：

# coding=utf-8

from selenium import webdriver

browser = webdriver.Chrome()
browser.get('https://www.taobao.com')
lis = browser.find_elements_by_css_selector('.service-bd li')
print(lis)
browser.close()

'''
[<selenium.webdriver.remote.webelement.WebElement (session="2e36b34a13572eb9b046174caf4fbf31", element="0.5182009301201136-1")>, <selenium.webdriver.remote.webelement.WebElement (session="2e36b34a13572eb9b046174caf4fbf31", element="0.5182009301201136-2")>, <selenium.webdriver.remote.webelement.WebElement (session="2e36b34a13572eb9b046174caf4fbf31", element="0.5182009301201136-3")>, <selenium.webdriver.remote.webelement.WebElement (session="2e36b34a13572eb9b046174caf4fbf31", element="0.5182009301201136-4")>, <selenium.webdriver.remote.webelement.WebElement (session="2e36b34a13572eb9b046174caf4fbf31", element="0.5182009301201136-5")>, <selenium.webdriver.remote.webelement.WebElement (session="2e36b34a13572eb9b046174caf4fbf31", element="0.5182009301201136-6")>, <selenium.webdriver.remote.webelement.WebElement (session="2e36b34a13572eb9b046174caf4fbf31", element="0.5182009301201136-7")>, <selenium.webdriver.remote.webelement.WebElement (session="2e36b34a13572eb9b046174caf4fbf31", element="0.5182009301201136-8")>, <selenium.webdriver.remote.webelement.WebElement (session="2e36b34a13572eb9b046174caf4fbf31", element="0.5182009301201136-9")>, <selenium.webdriver.remote.webelement.WebElement (session="2e36b34a13572eb9b046174caf4fbf31", element="0.5182009301201136-10")>, <selenium.webdriver.remote.webelement.WebElement (session="2e36b34a13572eb9b046174caf4fbf31", element="0.5182009301201136-11")>, <selenium.webdriver.remote.webelement.WebElement (session="2e36b34a13572eb9b046174caf4fbf31", element="0.5182009301201136-12")>, <selenium.webdriver.remote.webelement.WebElement (session="2e36b34a13572eb9b046174caf4fbf31", element="0.5182009301201136-13")>, <selenium.webdriver.remote.webelement.WebElement (session="2e36b34a13572eb9b046174caf4fbf31", element="0.5182009301201136-14")>, <selenium.webdriver.remote.webelement.WebElement (session="2e36b34a13572eb9b046174caf4fbf31", element="0.5182009301201136-15")>, <selenium.webdriver.remote.webelement.WebElement (session="2e36b34a13572eb9b046174caf4fbf31", element="0.5182009301201136-16")>]

'''

# coding=utf-8

from selenium import webdriver
from selenium.webdriver.common.by import By

browser = webdriver.Chrome()
browser.get('https://www.taobao.com')
lis = browser.find_elements(By.CSS_SELECTOR, '.service-bd li')
print(lis)
browser.close()


'''
[<selenium.webdriver.remote.webelement.WebElement (session="bbf860d905f8dab80e45492bd639e467", element="0.5137268514998159-1")>, <selenium.webdriver.remote.webelement.WebElement (session="bbf860d905f8dab80e45492bd639e467", element="0.5137268514998159-2")>, <selenium.webdriver.remote.webelement.WebElement (session="bbf860d905f8dab80e45492bd639e467", element="0.5137268514998159-3")>, <selenium.webdriver.remote.webelement.WebElement (session="bbf860d905f8dab80e45492bd639e467", element="0.5137268514998159-4")>, <selenium.webdriver.remote.webelement.WebElement (session="bbf860d905f8dab80e45492bd639e467", element="0.5137268514998159-5")>, <selenium.webdriver.remote.webelement.WebElement (session="bbf860d905f8dab80e45492bd639e467", element="0.5137268514998159-6")>, <selenium.webdriver.remote.webelement.WebElement (session="bbf860d905f8dab80e45492bd639e467", element="0.5137268514998159-7")>, <selenium.webdriver.remote.webelement.WebElement (session="bbf860d905f8dab80e45492bd639e467", element="0.5137268514998159-8")>, <selenium.webdriver.remote.webelement.WebElement (session="bbf860d905f8dab80e45492bd639e467", element="0.5137268514998159-9")>, <selenium.webdriver.remote.webelement.WebElement (session="bbf860d905f8dab80e45492bd639e467", element="0.5137268514998159-10")>, <selenium.webdriver.remote.webelement.WebElement (session="bbf860d905f8dab80e45492bd639e467", element="0.5137268514998159-11")>, <selenium.webdriver.remote.webelement.WebElement (session="bbf860d905f8dab80e45492bd639e467", element="0.5137268514998159-12")>, <selenium.webdriver.remote.webelement.WebElement (session="bbf860d905f8dab80e45492bd639e467", element="0.5137268514998159-13")>, <selenium.webdriver.remote.webelement.WebElement (session="bbf860d905f8dab80e45492bd639e467", element="0.5137268514998159-14")>, <selenium.webdriver.remote.webelement.WebElement (session="bbf860d905f8dab80e45492bd639e467", element="0.5137268514998159-15")>, <selenium.webdriver.remote.webelement.WebElement (session="bbf860d905f8dab80e45492bd639e467", element="0.5137268514998159-16")>]

'''

5.元素交互操作:

更多操作：https://selenium-python.readthedocs.io/api.html#module-selenium.webdriver.remote.webelement

# coding=utf-8

from selenium import webdriver
import time

browser = webdriver.Chrome()
browser.get('https://www.tmall.com')
input = browser.find_element_by_name('q')
input.send_keys('华为')
time.sleep(3)
input.clear()
input.send_keys('魅族')
button = browser.find_element_by_xpath(".//div[@id='mallSearch']//div[@class='mallSearch-input clearfix']/button")
button.click()
time.sleep(3)
browser.close()

6.交互动作：

更多操作：http://selenium-python.readthedocs.io/api.html#module-selenium.webdriver.common.action_chains

from selenium import webdriver
from selenium.webdriver import ActionChains

browser = webdriver.Chrome()
url = 'http://www.runoob.com/try/try.php?filename=jqueryui-api-droppable'
browser.get(url)
browser.switch_to.frame('iframeResult')
source = browser.find_element_by_css_selector('#draggable')
target = browser.find_element_by_css_selector('#droppable')
actions = ActionChains(browser)
actions.drag_and_drop(source, target)
actions.perform()

7.执行JavaScript

# coding=utf-8

from selenium import webdriver

browser = webdriver.Chrome()
browser.get('https://www.zhihu.com/explore')
browser.execute_script('window.scrollTo(0, document.body.scrollHeight)')
browser.execute_script('alert("To Bottom")')

8.获取元素信息：

（1）获取属性：get_attribute（'class'）

（2）获取文本值：

（3）获取 id ：

（4）获取 class：

（5）获取 location：

（6）获取 tag_name：

（7）获取 size：

# coding=utf-8

from selenium import webdriver

browser = webdriver.Chrome()
url = 'https://www.zhihu.com/explore'
browser.get(url)
logo = browser.find_element_by_id('zh-top-link-logo')
print(logo.get_attribute('class'))
print(logo.text)
print(logo.id)
print(logo.location)
print(logo.tag_name)
print(logo.size)


'''
zu-top-link-logo
知乎
0.5379235967183982-1
{'x': 6, 'y': 0}
a
{'height': 45, 'width': 61}
'''

9.Frame

import time
from selenium import webdriver
from selenium.common.exceptions import NoSuchElementException

browser = webdriver.Chrome()
url = 'http://www.runoob.com/try/try.php?filename=jqueryui-api-droppable'
browser.get(url)
browser.switch_to.frame('iframeResult')
source = browser.find_element_by_css_selector('#draggable')
print(source)
try:
    logo = browser.find_element_by_class_name('logo')
except NoSuchElementException:
    print('NO LOGO')
browser.switch_to.parent_frame()
logo = browser.find_element_by_class_name('logo')
print(logo)
print(logo.text)

10.等待

显式等待：在指定时间内等待某一条件，如满足直接返回，若到时间还不满足则抛异常

详细内容：http://selenium-python.readthedocs.io/api.html#module-selenium.webdriver.support.expected_conditions

• title_is 标题是某内容
• title_contains 标题包含某内容
• presence_of_element_located 元素加载出，传入定位元组，如(By.ID, 'p')
• visibility_of_element_located 元素可见，传入定位元组
• visibility_of 可见，传入元素对象
• presence_of_all_elements_located 所有元素加载出
• text_to_be_present_in_element 某个元素文本包含某文字
• text_to_be_present_in_element_value 某个元素值包含某文字
• frame_to_be_available_and_switch_to_it frame加载并切换
• invisibility_of_element_located 元素不可见
• element_to_be_clickable 元素可点击
• staleness_of 判断一个元素是否仍在DOM，可判断页面是否已经刷新
• element_to_be_selected 元素可选择，传元素对象
• element_located_to_be_selected 元素可选择，传入定位元组
• element_selection_state_to_be 传入元素对象以及状态，相等返回True，否则返回False
• element_located_selection_state_to_be 传入定位元组以及状态，相等返回True，否则返回False
• alert_is_present 是否出现Alert

隐式等待：直接指定等待多久再去获取对象

from selenium import webdriver

browser = webdriver.Chrome()
browser.implicitly_wait(10)
browser.get('https://www.zhihu.com/explore')
input = browser.find_element_by_class_name('zu-top-add-question')
print(input)

11.前进后退

import time
from selenium import webdriver

browser = webdriver.Chrome()
browser.get('https://www.baidu.com/')
browser.get('https://www.taobao.com/')
browser.get('https://www.python.org/')
browser.back()
time.sleep(1)
browser.forward()
browser.close()

12.Cookies：获取，添加和删除cookies

from selenium import webdriver

browser = webdriver.Chrome()
browser.get('https://www.zhihu.com/explore')
print(browser.get_cookies())
browser.add_cookie({'name': 'name', 'domain': 'www.zhihu.com', 'value': 'germey'})
print(browser.get_cookies())
browser.delete_all_cookies()
print(browser.get_cookies())

13.选项卡管理：

import time
from selenium import webdriver

browser = webdriver.Chrome()
browser.get('https://www.baidu.com')
browser.execute_script('window.open()')
print(browser.window_handles)
browser.switch_to_window(browser.window_handles[1])
browser.get('https://www.taobao.com')
time.sleep(1)
browser.switch_to_window(browser.window_handles[0])
browser.get('https://python.org')

14.异常处理

详细文档：http://selenium-python.readthedocs.io/api.html#module-selenium.common.exceptions

from selenium import webdriver
from selenium.common.exceptions import TimeoutException, NoSuchElementException

browser = webdriver.Chrome()
try:
    browser.get('https://www.baidu.com')
except TimeoutException:
    print('Time Out')
try:
    browser.find_element_by_id('hello')
except NoSuchElementException:
    print('No Element')
finally:
    browser.close()


'''
No Element
'''