selenium python实例_Python数据分析及可视化实例之Selenium、PantomJS

一直以来我觉得用在运维的Selenium、PantomJS是一个重器，

不到万不得已的时候不要祭出这个大杀器，

但是涉及到JavaScript及Ajax渲染的时候，Requests就完全懵逼了！

最近回过头来重新审视这货，

这个重器用反倒轻便了很多。

1.安装Selenium、PantomJS

Selenium可以直接通过pip安装，PantomJS则是一个exe可执行文件，需要下载解压。在使用的时候指定exe的绝对路径即可。

2.Selenium、PantomJS基本设置

from selenium import webdriver

from selenium.webdriver.common.desired_capabilities import DesiredCapabilities

dcap = DesiredCapabilities.PHANTOMJS

dcap[ "phantomjs.page.settings.userAgent"] = "Mozilla / 4.0(Windows NT 10.0; Win64;x64) AppleWebKit / 537.36(KHTML, like Gecko) Chrome/51.0.2704.79 Safari/ 537.36Edge/14.14393"

# 请求头不一样，自适应的窗口不一样，卧槽，坑爹！

driver = webdriver.PhantomJS(desired_capabilities=dcap)

driver.set_page_load_timeout(10)

driver.set_script_timeout(10) # 设置页面退出时间，没有必要等一个网页加载完了采集

# 采集网页源码

try:

driver.get(inurl)

content = driver.page_source

# print(content)

time.sleep(1)

except:

driver.execute_script('window.stop()')

driver.close()

3.Selenium、PantomJS基本操作

如果你的网络和机子足够好，基本上就不用等待网页渲染，

否则，还需要等待，如果用time.sleep()，则有点笨拙，

#等待页面渲染完成

from selenium.webdriver.common.by import By

from selenium.webdriver.support.ui import WebDriverWait

from selenium.webdriver.support import expected_conditions as EC

...

try:

element = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.ID, "loadedButton")))

# 等某个标签元素出来，不见鸭子不撒鹰。

finally: # 撒鹰

print(driver.find_element_by_id("content").text)

driver.close()

或者用

try:

elem == driver.find_element_by_tag_name("html")

# 抛出StaleElementReferenceException异常说明elem元素已经消失了，也就说明页面已经跳转了。

except StaleElementReferenceException:

return

其他driver内置函数，可以通过查看源代码或者在pycharm提示获取。

4.Xpath定位Html标签

1.id定位：find_element_by_id(self, id_)

2.name定位：find_element_by_name(self, name)

3.class定位：find_element_by_class_name(self, name)

4.tag定位：find_element_by_tag_name(self, name)

5.link定位：find_element_by_link_text(self, link_text)

6.partial_link定位find_element_by_partial_link_text(self, link_text)

7.xpath定位：find_element_by_xpath(self, xpath)

8.css定位：find_element_by_css_selector(self, css_selector)

9.id复数定位find_elements_by_id(self, id_)

10.name复数定位find_elements_by_name(self, name)

11.class复数定位find_elements_by_class_name(self, name)

12.tag复数定位find_elements_by_tag_name(self, name)

13.link复数定位find_elements_by_link_text(self, text)

14.partial_link复数定位find_elements_by_partial_link_text(self, link_text)

15.xpath复数定位find_elements_by_xpath(self, xpath)

16.css复数定位find_elements_by_css_selector(self, css_selector

17.find_element(self, by='id', value=None)

18.find_elements(self, by='id', value=None)

其中element方法定位到是是单数，是直接定位到元素；elements方法是复数，这个学过英文的都知道，定位到的是一组元素，返回的是list队列。可参照Re函数中的findall理解。

如果定位不了标签，只能上JS大法：Selenium2+python自动化46-js解决click失效问题 - 上海-悠悠 - 博客园该作者已经出书，很便宜，可以考虑入一本。如果还想深入练习本节大法，同时推荐博客园ID：虫师虫师 - 博客园，这位兄台是做运维的，我最初就是用他的PDF教材入门Selenium，而且也已经出书了，是不是听这ID，就不是一般的牛逼，哈哈...

5.完整例子

这个例子属于标准化操作，在实际中可以适当简化，并结合上面的Xpath定位完成。

from selenium import webdriver

import time

from selenium.webdriver.common.desired_capabilities import DesiredCapabilities

dcap = dict(DesiredCapabilities.PHANTOMJS)

dcap["phantomjs.page.settings.userAgent"] = "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36"

driver = webdriver.PhantomJS(executable_path=r'C:\Users\taojw\Desktop\pywork\phantomjs-2.1.1-windows\bin\phantomjs.exe', desired_capabilities=dcap)

driver.get("http://pythonscraping.com/pages/javascript/ajaxDemo.html")

time.sleep(3)

print(driver.find_element_by_id("content").text)

driver.close()

#设置PHANTOMJS的USER-AGENT

from selenium import webdriver

from selenium.webdriver.common.desired_capabilities import DesiredCapabilities

dcap = dict(DesiredCapabilities.PHANTOMJS)

dcap["phantomjs.page.settings.userAgent"] = "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36"

driver = webdriver.PhantomJS(executable_path='./phantomjs.exe', desired_capabilities=dcap)

driver.get("http://dianping.com/")

cap_dict = driver.desired_capabilities #查看所有可用的desired_capabilities属性。

for key in cap_dict:

print('%s:%s' % (key, cap_dict[key]))

print(driver.current_url)

driver.quit()

#等待页面渲染完成

from selenium.webdriver.common.by import By

from selenium.webdriver.support.ui import WebDriverWait

from selenium.webdriver.support import expected_conditions as EC

driver = webdriver.PhantomJS(executable_path=r'C:\Users\taojw\Desktop\pywork\phantomjs-2.1.1-windows\bin\phantomjs.exe')

driver.get("http://pythonscraping.com/pages/javascript/ajaxDemo.html")

try:

element = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.ID, "loadedButton")))

finally:

print(driver.find_element_by_id("content").text)

driver.close()

#处理Javascript重定向

from selenium import webdriver

import time

from selenium.webdriver.remote.webelement import WebElement

from selenium.common.exceptions import StaleElementReferenceException

def waitForLoad(driver):

elem = driver.find_element_by_tag_name("html")

count = 0

while True:

count += 1

if count > 20:

print("Timing out after 10 seconds and returning")

return

time.sleep(.5)

try:

elem == driver.find_element_by_tag_name("html")

except StaleElementReferenceException:

return

driver = webdriver.PhantomJS(executable_path=r'C:\Users\taojw\Desktop\pywork\phantomjs-2.1.1-windows\bin\phantomjs.exe')

driver.get("http://pythonscraping.com/pages/javascript/redirectDemo1.html")

waitForLoad(driver)

print(driver.page_source)

######

from selenium import webdriver

from selenium.webdriver.remote.webelement import WebElement

from selenium.webdriver import ActionChains

driver = webdriver.PhantomJS(executable_path='phantomjs/bin/phantomjs')

driver.get('http://pythonscraping.com/pages/javascript/draggableDemo.html')

print(driver.find_element_by_id("message").text)

element = driver.find_element_by_id("draggable")

target = driver.find_element_by_id("div2")

actions = ActionChains(driver)

actions.drag_and_drop(element, target).perform()

print(driver.find_element_by_id("message").text)

#######

#截屏

driver.get_screenshot_as_file('tmp/pythonscraping.png')

####

#登陆知乎，然后能自动点击页面下方的“更多”，以载入更多的内容

from selenium import webdriver

from selenium.webdriver.common.keys import Keys

from selenium.webdriver.support.ui import WebDriverWait

from selenium.webdriver import ActionChains

import time

import sys

driver = webdriver.PhantomJS(executable_path='C:\Users\Gentlyguitar\Desktop\phantomjs-1.9.7-windows\phantomjs.exe')

driver.get("http://www.zhihu.com/#signin")

#driver.find_element_by_name('email').send_keys('your email')

driver.find_element_by_xpath('//input[@name="password"]').send_keys('your password')

#driver.find_element_by_xpath('//input[@name="password"]').send_keys(Keys.RETURN)

time.sleep(2)

driver.get_screenshot_as_file('show.png')

#driver.find_element_by_xpath('//button[@class="sign-button"]').click()

driver.find_element_by_xpath('//form[@class="zu-side-login-box"]').submit()

try:

#等待页面加载完毕

dr=WebDriverWait(driver,5)

dr.until(lambda the_driver:the_driver.find_element_by_xpath('//a[@class="zu-top-nav-userinfo "]').is_displayed())

except:

print('登录失败')

sys.exit(0)

driver.get_screenshot_as_file('show.png')

#user=driver.find_element_by_class_name('zu-top-nav-userinfo ')

#webdriver.ActionChains(driver).move_to_element(user).perform() #移动鼠标到我的用户名

loadmore=driver.find_element_by_xpath('//a[@id="zh-load-more"]')

actions = ActionChains(driver)

actions.move_to_element(loadmore)

actions.click(loadmore)

actions.perform()

time.sleep(2)

driver.get_screenshot_as_file('show.png')

print(driver.current_url)

print(driver.page_source)

driver.quit()

##################################################################################

关于本节更多的实战技巧将会放在实例中。

新手可查阅历史目录：yeayee：Python数据分析及可视化实例目录​zhuanlan.zhihu.com

最后，别只收藏不关注哈