实战
实战大项目:模拟登录丁香园,并抓取论坛页面所有的人员基本信息与回复帖子内容。丁香园论坛:http://www.dxy.cn/bbs/thread/626626#626626 。
解析
默认的登录方式是扫码登录,所以要切换成账号密码登录:
- 通过selenium定位元素登录链接
login_link = driver.find_element_by_link_text(‘登录’) - 把login_link元素传入JavaScript语句,代替
document.getElementsByClassName(“site-nav-right fr”)[0].childNodes[1],而arguments[0]可以帮我们把selenium的元素传入到JavaScript语句中。arguments指的是execute_script()方法中js代码后面的所有参数,arguments[0]表示第一个参数,argument[1]表示第二个参数
js='arguments[0].removeAttribute(argument[1])'
driver.execute_script(js,login_link,”target”)
- 点击登陆链接
login_link.click()
完整版代码:
# -*- coding:utf-8 -*-
import requests, json, re, random,time
from bs4 import BeautifulSoup
from selenium import webdriver
from lxml import etree
class getUrl(object):
def __init__(self):
self.headers={
"Connection": "keep-alive",
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/51.0.2704.63 Safari/537.36",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
"Accept-Encoding": "gzip, deflate, sdch",
"Accept-Language": "zh-CN,zh;q=0.8"
};
def run(self):
browser = webdriver.Chrome()
browser.get('https://auth.dxy.cn/accounts/login?service=http://www.dxy.cn/bbs/index.html')
time.sleep(1)
#切换账号密码登录表单
js1 = 'document.querySelector("#j_loginTab1").style.display="none";'
browser.execute_script(js1)
time.sleep(1)
js2 = 'document.querySelector("#j_loginTab2").style.display="block";'
browser.execute_script(js2)
#输入账号密码
input_name = browser.find_element_by_name('username')
input_name.clear()
input_name.send_keys('......')
input_pass = browser.find_element_by_name('password')
input_pass.clear()
input_pass.send_keys('......')
browser.find_element_by_xpath('//*[@class="form__button"]/button').click()
#此步骤应该有验证码,先跳过
time.sleep(10)
cookie = browser.get_cookies()
cookie_dict = {i['name']:i['value'] for i in cookie}
#转到抓取页面
browser.get("http://www.dxy.cn/bbs/thread/626626#626626");
html = browser.page_source
tree = etree.HTML(html)
user = tree.xpath('//div[@id="postcontainer"]//div[@class="auth"]/a/text()')
content = tree.xpath('//td[@class="postbody"]')
for i in range(0,len(user)):
result = user[i].strip()+":"+content[i].xpath('string(.)').strip()
#写入文件
dir_file = open("DXY_records.txt",'a', encoding="utf-8")
dir_file.write(result+"\n")
dir_file.write('-' * 80+"\n")
dir_file.close()
print('-' * 5 +"抓取结束"+'-' * 5)
if __name__ == '__main__':
geturl = getUrl()
geturl.run()
结果: