最帅爬虫_斗鱼 selenium 翻页

网址： https://www.douyu.com/g_LOL

需求：获取所有在线直播的房间信息

代码实现：

import time
from lxml import etree
from selenium import webdriver
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By

# def get_content_by_selenium(url):
#     '''
#     发送请求，获取数据（测试用）
#     '''
#     driver.get(url)
#     #先用强制等待，显示等待主要的作用就是后期优化
#     time.sleep(3)
#     html_str = driver.page_source
#     print(html_str)
#     return html_str

def pase_page(html_str):
    '''
    解析一页的直播信息
    '''
    tree = etree.HTML(html_str)
    li_list = tree.xpath('//ul[@class="layout-Cover-list"]/li')
    for li in li_list:
        live_title = li.xpath('.//h3/text()')[0]
        live_type = li.xpath('.//div[@class="DyListCover-info"]/span/text()')[0]
        people_tag = li.xpath('.//div[@class="DyListCover-content"]/span/text()')
        if people_tag:
            people_tag = people_tag[0]
        else:
            people_tag = ''

        live_people = li.xpath('.//h2[@class="DyListCover-user"]/text()')[0]
        # print(live_people)
        live_hot_num = li.xpath('.//span[@class="DyListCover-hot"]/text()')[0]
        # print(live_hot_num)
        item = {}
        item['live_title'] = live_title
        item['live_type'] = live_type
        item['people_tag'] = people_tag
        item['live_people'] = live_people
        item['live_hot_num'] = live_hot_num
        print(item)

def main():
    # 1、确定url
    base_url ='https://www.douyu.com/g_LOL'

    # 2、使用slenium请求获取数据
    # html_str = get_content_by_selenium(base_url)
    # 使用selenium第二步
    driver.get(base_url)

    # 等待
    time.sleep(3)

    while True:
        # 获取当前driver的page_source
        # 第四步
        html_str = driver.page_source
        # print(html_str)

        # 3、解析
        pase_page(html_str)

        # 4、获取下一页的按钮
        next_page = driver.find_element_by_xpath('//ul[@class="dy-Pagination ListPagination"]/li[@title="下一页"]')

        # 5、获取aria-disabled属性
        aria_disabled = next_page.get_attribute('aria-disabled')
        # print(aria_disabled)
        if aria_disabled == 'true':
            break
            # pass
        # 终止条件：aria-disabled="true"--》不用再点击下一页了

        # 6、点击下一页
        next_page.click()

        # 7、等待
        wait.until(EC.presence_of_all_elements_located((By.XPATH,'//ul[@class="layout-Cover-list"]/li')))

if __name__ == '__main__':
    # 使用selenium的第一步：在全局变量中创建一个driver对象
    # 解决优化的问题
    driver = webdriver.Chrome()
    wait = WebDriverWait(driver,20)
    main()