抓取豆瓣帖子信息

# coding = 'utf-8'
import requests,time,random,csv
from lxml import etree
from selenium import webdriver
from selenium.webdriver import ActionChains
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.common.by import By
ua = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36'
headers = {'User-Agent': ua}

def get_tracks(distance):
    """
    拿到移动轨迹，模仿人的滑动行为，先匀加速后匀减速
    匀变速运动基本公式：
    ①v = v0+at
    ②s = v0t+1/2at^2
    """

    # 初速度
    v = 0
    # 单位时间为0.3s来统计轨迹，轨迹即0.3内的位移
    t = 0.3
    # 位置/轨迹列表，列表内的一个元素代表0.3s的位移
    tracks = []
    # 当前位移
    current = 0
    # 到达mid值开始减速
    mid = distance * 4 / 5
    while current < distance:
        if current < mid:
            # 加速度越小，单位时间内的位移越小，模拟的轨迹就越多越详细
            a = 2
        else:
            a = -3
        # 初速度
        v0 = v
        # 0.3秒内的位移
        s = v0 * t + 0.5 * a * (t ** 2)
        # 当前的位置
        current += s
        # 添加到轨迹列表
        tracks.append(round(s))
        # 速度已经到达v,该速度作为下次的初速度
        v = v0 + a * t
    return tracks


#1.打开豆瓣网,并且最大化
#
#加表头
options = webdriver.ChromeOptions()
# options.add_argument('--headless')
options.add_argument('--user-agent={}'.format(ua))
browser = webdriver.Chrome(options=options)
#
browser.maximize_window()
wait = WebDriverWait(browser, 10)
browser.get('https://www.douban.com/')
#
# 2.默认验证码登录页面,切换至密码登录框架,点击切换密码登录
browser.switch_to.frame(browser.find_element(By.TAG_NAME,'iframe'))
wait.until(lambda browser:browser.find_element(By.XPATH,'//ul[@class="tab-start"]/li[2]')).click()
time.sleep(3)
# 停顿3秒后,输入用户名和密码,点击登录按钮
wait.until(lambda browser:browser.find_element(By.XPATH,'//*[@id="username"]')).send_keys("username")
wait.until(lambda browser:browser.find_element(By.XPATH,'//*[@id="password"]')).send_keys("password")
wait.until(lambda browser:browser.find_element(By.CLASS_NAME,'account-form-field-submit')).click()
time.sleep(3)
#
#5.按住开始滑动位置按钮-先移动180个像素
browser.switch_to.frame(browser.find_element(By.ID,'tcaptcha_iframe'))
element = browser.find_element(By.XPATH,'//*[@id="tcaptcha_drag_thumb"]')
ActionChains(browser).click_and_hold(on_element=element).perform()
#move_to_element_with_offset().移动到距离某个元素左上角多少距离的位置
ActionChains(browser).move_to_element_with_offset(to_element=element,xoffset=180,yoffset=0).perform()
#
#6.使用加速度函数移动剩下的距离
tracks = get_tracks(23)
for track in tracks:
#开始移动move_by_offset()
ActionChains(browser).move_by_offset(xoffset=track,yoffset=0).perform()
#
#7.延迟释放鼠标：release()
time.sleep(0.5)
ActionChains(browser).release().perform()
#
#7.获得cookies后,退出浏览器
time.sleep(10)
cookies=browser.get_cookies()
browser.quit()
print(cookies)
cookies={i['name']:i['value'] for i in cookies}

def get_txt(url,count = 0,max_count = 2):
    try:
        time.sleep(random.random())
        r = requests.get(url=url,headers = headers,cookies = cookies,timeout = 30)
        r.raise_for_status()
        r.encoding = 'utf-8'
        # print(r.text)
        xml = etree.HTML(r.text)
        return xml
    except:
        count += 1
        print(f'第{count}次爬取{url}失败')
        time.sleep(10)
        if count < max_count:
            return get_txt(url = url,count = count)

def main():
    pass
if __name__ == '__main__':
    base_url = ''
    main()