网址: https://www.douyu.com/g_LOL
需求:获取所有在线直播的房间信息
代码实现:
import time
from lxml import etree
from selenium import webdriver
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
# def get_content_by_selenium(url):
# '''
# 发送请求,获取数据(测试用)
# '''
# driver.get(url)
# #先用强制等待,显示等待主要的作用就是后期优化
# time.sleep(3)
# html_str = driver.page_source
# print(html_str)
# return html_str
def pase_page(html_str):
'''
解析一页的直播信息
'''
tree = etree.HTML(html_str)
li_list = tree.xpath('//ul[@class="layout-Cover-list"]/li')
for li in li_list:
live_title = li.xpath('.//h3/text()')[0]
live_type = li.xpath('.//div[@class="DyListCover-info"]/span/text()')[0]
people_tag = li.xpath('.//div[@class="DyListCover-content"]/span/text()')
if people_tag:
people_tag = people_tag[0]
else:
people_tag = ''
live_people = li.xpath('.//h2[@class="DyListCover-user"]/text()')[0]
# print(live_people)
live_hot_num = li.xpath('.//span[@class="DyListCover-hot"]/text()')[0]
# print(live_hot_num)
item = {}
item['live_title'] = live_title
item['live_type'] = live_type
item['people_tag'] = people_tag
item['live_people'] = live_people
item['live_hot_num'] = live_hot_num
print(item)
def main():
# 1、确定url
base_url ='https://www.douyu.com/g_LOL'
# 2、使用slenium请求获取数据
# html_str = get_content_by_selenium(base_url)
# 使用selenium第二步
driver.get(base_url)
# 等待
time.sleep(3)
while True:
# 获取当前driver的page_source
# 第四步
html_str = driver.page_source
# print(html_str)
# 3、解析
pase_page(html_str)
# 4、获取下一页的按钮
next_page = driver.find_element_by_xpath('//ul[@class="dy-Pagination ListPagination"]/li[@title="下一页"]')
# 5、获取aria-disabled属性
aria_disabled = next_page.get_attribute('aria-disabled')
# print(aria_disabled)
if aria_disabled == 'true':
break
# pass
# 终止条件:aria-disabled="true"--》不用再点击下一页了
# 6、点击下一页
next_page.click()
# 7、等待
wait.until(EC.presence_of_all_elements_located((By.XPATH,'//ul[@class="layout-Cover-list"]/li')))
if __name__ == '__main__':
# 使用selenium的第一步:在全局变量中创建一个driver对象
# 解决优化的问题
driver = webdriver.Chrome()
wait = WebDriverWait(driver,20)
main()