第十章 采集javascript 使用selenium库进行获取

#!/usr/bin/env python
# _*_ coding:utf-8 _*_
# 等3秒获取指定的内容 会有selenium不支持无头的phantomjs的警告
# from selenium import webdriver
# import time
# driver=webdriver.PhantomJS(executable_path='D:/pycharm/phantomjs-2.1.1-windows/bin/phantomjs')
# driver.get("http://pythonscraping.com/pages/javascript/ajaxDemo.html")
# time.sleep(3)
# print(driver.find_element_by_id('content').text)
# driver.close()


# 用id检查页面是不是已经完全加载
# from selenium import webdriver
# from selenium.webdriver.common.by import By
# from selenium.webdriver.support import expected_conditions as EC
# from selenium.webdriver.support.ui import WebDriverWait
#
# driver=webdriver.PhantomJS(executable_path='D:/pycharm/phantomjs-2.1.1-windows/bin/phantomjs')
# driver.get("http://pythonscraping.com/pages/javascript/ajaxDemo.html")
# try:
#     element=WebDriverWait(driver,10).until(
#         EC.presence_of_element_located(By.ID,"loadedButton"))
# finally:
#     print(driver.find_element_by_id("content").text)
#     driver.close()

# 处理重定向 客户端重定向的处理 在页面开始加载时监控dom元素，重复调用这个元素
# 直到selenium抛出StaleElementReferenceException异常，说明网页发生了跳转

# 每半分钟检查一次网页，看是否html的标签还在不在，时限为10s
import time

from selenium import webdriver
from selenium.common.exceptions import StaleElementReferenceException


def waitForLoad(driver):
    elem=driver.find_element_by_tag_name("html")
    count=0
    while True:
        count+=1
        if count>20:
            print("timing put after 10 seconds and returns")
            return
        time.sleep(.5)
        try:
            elem==driver.find_element_by_tag_name("html")
        except StaleElementReferenceException:
            return
driver=webdriver.phantomjs(executable_path='D:/pycharm/phantomjs-2.1.1-windows/bin/phantomjs')
driver.get("http://pythonscraping.com/pages/javascript/redirectDemo1.html")
waitForLoad(driver)
print(driver.page_source)