python selenium，PhantomJS运用抓取滚动条滚动加载的页面， js动作操作，模拟登陆...

最新推荐文章于 2023-12-06 23:45:00 发布

weixin_33887443

最新推荐文章于 2023-12-06 23:45:00 发布

阅读量304

点赞数

文章标签： python

原文链接：https://yq.aliyun.com/articles/70045

版权

1、运用

#!/usr/bin/python
#encoding=utf-8
import  sys
from selenium import webdriver
reload(sys)
sys.setdefaultencoding( "utf-8" )
driver = webdriver.PhantomJS(executable_path='/home/lhy/phantomjs-1.9.8-linux-x86_64/bin/phantomjs')
driver.get("http://item.jd.com/2914823.html")
#driver.find_element_by_id('search_form_input_homepage').send_keys("Nirvana")
#driver.find_element_by_id("search_button_homepage").click()

print driver.page_source
fo = open("aaaa1.txt", "wb")
fo.write(driver.page_source)
fo.close()
driver.quit()

2、抓取下拉加载的页面

#coding=utf-8
import requests
import  re
import  time
from pyquery import PyQuery as pq
from lxml import etree
from bs4 import BeautifulSoup
import sys
from selenium import webdriver
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
reload(sys)
sys.setdefaultencoding("utf-8")

urls=[]

def getHtml2(url):
    user_agent = 'Mozilla/4.0 (compatible; MSIE 5.5; windows NT)'
    headers = {'User-Agent': user_agent}
    r = requests.post(url, headers=headers)
    fo = open("phonesinfo1.txt", "wb")
    fo.write(r.content)
    fo.close()
    #print r.content
    return r.content

def getHtml(url):
    dcap = dict(DesiredCapabilities.PHANTOMJS)
    dcap["phantomjs.page.settings.userAgent"] = (
        "Mozilla/4.0 (compatible; MSIE 5.5; windows NT)"
    )

    driver = webdriver.PhantomJS(desired_capabilities=dcap)
    #driver = webdriver.PhantomJS(executable_path='/home/lhy/phantomjs-1.9.8-linux-x86_64/bin/phantomjs')
    #driver=webdriver.Chrome()
    driver.get(url)
   <span style="color:#FF0000;"> js="document.body.scrollTop=1000"#滚动条下拉1000px
    driver.execute_script(js)</span>
    driver.implicitly_wait(30)
    #time.sleep(5)
    #fo = open("phonesinfo2.txt", "wb")
    #fo.write(driver.page_source)
    #fo.close()
    html=driver.page_source
    driver.quit()
    return html

def getPqHtml(html):
    pqHtml = pq(html)
    return pqHtml
def getUrlsFromFile(fileUrl):
    with open('phoneurl.txt', 'r') as f:
        lines = f.readlines()
        for line in lines:
            url_one = line.strip()
            print url_one
            urls.append(url_one)


url="http://localhost:8080/pro/html.html"
text=getHtml(url)
fo = open("taobao2.txt", "wb")
fo.write(text)
fo.close()
print text

html 页面

<html>
<head>
  
</head>
<body style="height:5000px">
<div id="top_div" style="display:none">ffffffffffffffffffffff</div>
<script>
//document.body.scrollTop=10000;
window.onscroll = function(){
    var t = document.documentElement.scrollTop || document.body.scrollTop; 
    var top_div = document.getElementById( "top_div" );
    if( t >= 300 ) {
        //  alert(t);
        top_div.style.display = "block";
    }
  // else { top_div.style.display = "none";}
}
</script>

</body>
</html>

3、模拟登陆

# coding = utf-8
from selenium import webdriver

browser = webdriver.Firefox()

<span style="color:#FF0000;">browser.get("http://localhost:8080/pro")
browser.find_element_by_name("password").clear() #先清除文本框上密码

browser.find_element_by_name("username").send_keys("test")  #设置值
browser.find_element_by_name("password").send_keys("123")  #设置值

yzm=browser.find_element_by_class_name("yzm-img").find_element_by_tag_name("span").text  #获取验证码值
yzm=yzm.replace(' ','') #清除空格
browser.find_element_by_class_name("yzm-sr").send_keys(yzm) #设置验证码
browser.find_element_by_id("tijiao").click() #点击按钮 提交表单

print browser.current_url 
browser.get("http://localhost:8080/pro/test.jsp")#模拟登陆成功后会自动把cookie保存在对象中，对需认证页面可直接访问</span>
print browser.page_source
#browser.quit()

4、百度搜索

# coding = utf-8
from selenium import webdriver

browser = webdriver.Firefox()

browser.get("http://www.baidu.com")
browser.find_element_by_id("kw").clear()
browser.find_element_by_id("kw").send_keys("selenium")
browser.find_element_by_id("su").click()

print browser.current_url  #点击成功后调转页面的url
#browser.quit()