# coding = 'utf-8'
import requests,time,random,csv
from lxml import etree
from selenium import webdriver
from selenium.webdriver import ActionChains
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.common.by import By
ua = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36'
headers = {'User-Agent': ua}
def get_tracks(distance):
"""
拿到移动轨迹,模仿人的滑动行为,先匀加速后匀减速
匀变速运动基本公式:
①v = v0+at
②s = v0t+1/2at^2
"""
# 初速度
v = 0
# 单位时间为0.3s来统计轨迹,轨迹即0.3内的位移
t = 0.3
# 位置/轨迹列表,列表内的一个元素代表0.3s的位移
tracks = []
# 当前位移
current = 0
# 到达mid值开始减速
mid = distance * 4 / 5
while current < distance:
if current < mid:
# 加速度越小,单位时间内的位移越小,模拟的轨迹就越多越详细
a = 2
else:
a = -3
# 初速度
v0 = v
# 0.3秒内的位移
s = v0 * t + 0.5 * a * (t ** 2)
# 当前的位置
current += s
# 添加到轨迹列表
tracks.append(round(s))
# 速度已经到达v,该速度作为下次的初速度
v = v0 + a * t
return tracks
#1.打开豆瓣网,并且最大化
#
#加表头
options = webdriver.ChromeOptions()
# options.add_argument('--headless')
options.add_argument('--user-agent={}'.format(ua))
browser = webdriver.Chrome(options=options)
#
browser.maximize_window()
wait = WebDriverWait(browser, 10)
browser.get('https://www.douban.com/')
#
# 2.默认验证码登录页面,切换至密码登录框架,点击切换密码登录
browser.switch_to.frame(browser.find_element(By.TAG_NAME,'iframe'))
wait.until(lambda browser:browser.find_element(By.XPATH,'//ul[@class="tab-start"]/li[2]')).click()
time.sleep(3)
# 停顿3秒后,输入用户名和密码,点击登录按钮
wait.until(lambda browser:browser.find_element(By.XPATH,'//*[@id="username"]')).send_keys("username")
wait.until(lambda browser:browser.find_element(By.XPATH,'//*[@id="password"]')).send_keys("password")
wait.until(lambda browser:browser.find_element(By.CLASS_NAME,'account-form-field-submit')).click()
time.sleep(3)
#
#5.按住开始滑动位置按钮-先移动180个像素
browser.switch_to.frame(browser.find_element(By.ID,'tcaptcha_iframe'))
element = browser.find_element(By.XPATH,'//*[@id="tcaptcha_drag_thumb"]')
ActionChains(browser).click_and_hold(on_element=element).perform()
#move_to_element_with_offset().移动到距离某个元素左上角多少距离的位置
ActionChains(browser).move_to_element_with_offset(to_element=element,xoffset=180,yoffset=0).perform()
#
#6.使用加速度函数移动剩下的距离
tracks = get_tracks(23)
for track in tracks:
#开始移动move_by_offset()
ActionChains(browser).move_by_offset(xoffset=track,yoffset=0).perform()
#
#7.延迟释放鼠标:release()
time.sleep(0.5)
ActionChains(browser).release().perform()
#
#7.获得cookies后,退出浏览器
time.sleep(10)
cookies=browser.get_cookies()
browser.quit()
print(cookies)
cookies={i['name']:i['value'] for i in cookies}
def get_txt(url,count = 0,max_count = 2):
try:
time.sleep(random.random())
r = requests.get(url=url,headers = headers,cookies = cookies,timeout = 30)
r.raise_for_status()
r.encoding = 'utf-8'
# print(r.text)
xml = etree.HTML(r.text)
return xml
except:
count += 1
print(f'第{count}次爬取{url}失败')
time.sleep(10)
if count < max_count:
return get_txt(url = url,count = count)
def main():
pass
if __name__ == '__main__':
base_url = ''
main()
抓取豆瓣帖子信息
于 2022-01-08 00:45:27 首次发布