爬虫-斗鱼直播

最新推荐文章于 2021-09-11 21:20:25 发布

一岁一骷髅

最新推荐文章于 2021-09-11 21:20:25 发布

阅读量172

点赞数

分类专栏：爬虫

本文链接：https://blog.csdn.net/weixin_42766128/article/details/101305722

版权

爬虫专栏收录该内容

12 篇文章 0 订阅

订阅专栏

from selenium import webdriver
import time
from lxml import etree

driver = webdriver.PhantomJS(executable_path=r'F:\老师发的文件\系统班第四阶段\day923\phantomjs-2.1.1-windows\bin\phantomjs.exe')
def request_html(url):
    driver.get(url=url)
    time.sleep(3)
    driver.save_screenshot('douyu.png')
    with open('douyu.html','w',encoding='utf-8') as f:
        f.write(driver.page_source)


def parse_html(response):
    tree = etree.HTML(response)
    li_list = tree.xpath('//div[@class="layout-Module-container layout-Cover ListContent"]/ul/li')
    # print(len(li_list))
    for li in li_list:
        type = li.xpath('.//div[@class="DyListCover-content"]/div[1]/span/text()')[0]
        title = li.xpath('.//div[@class="DyListCover-content"]/div[1]/h3/text()')[0]
        hot = li.xpath('.//div[@class="DyListCover-content"]/div[2]/span/text()')[0]
        username = li.xpath('.//div[@class="DyListCover-content"]/div[2]/h2/text()')[0]
        print(type,'--',title,'--',hot,'--',username)
    return tree
if __name__ == '__main__':
    url = 'https://www.douyu.com/directory/all'
    # 建立连接
    request_html(url)
    # 返回结果
    response = driver.page_source
    # 获取数据
    tree = parse_html(response)
    page = 1
    # 点击下一页
    while True:
        print('==============={}==================='.format(page))
        flag = tree.xpath('//li[@title="下一页"]/@aria-disabled')[0]
        if flag == 'true':
            break
        driver.find_element_by_class_name(' dy-Pagination-next').click()
        time.sleep(3)
        response = driver.page_source
        tree = parse_html(response)
        page += 1