python之b站top100视频的分类爬取

我爱寿司

已于 2025-03-28 22:13:27 修改

阅读量743

点赞数 8

文章标签：信息可视化 python

于 2024-10-09 19:46:24 首次发布

本文链接：https://blog.csdn.net/weixin_57142950/article/details/142794471

版权

import csv
from selenium import webdriver
import pandas as pd
from selenium.webdriver.common.by import By


if __name__ == '__main__':

    url = 'https://www.bilibili.com/v/popular/rank/game'  #末尾的game是类别，可根据需求修稿
    driver = webdriver.Chrome()
    driver.get(url)

    csv_file = "data/top100_url-游戏.csv"

    with open(csv_file, 'a', newline='', encoding='utf-8') as f:
        writer = csv.writer(f)
        writer.writerow(['b站实时排行榜前一百视频url', 'up主昵称'])

        i = 1
        print()
        while (i < 101):
            all_datas = driver.find_elements(By.XPATH,f'//*[@id="app"]/div/div[2]/div[2]/ul/li[{i}]/div/div[2]/a')
                                             #f'//*[@id="app"]/div/div[2]/div[2]/ul/li[{i}]/div/div[2]/a
            all_up_name = driver.find_elements(By.XPATH,
                f'//*[@id="app"]/div/div[2]/div[2]/ul/li[{i}]/div/div[2]/div/a/span')
            href_values = [element.get_attribute("href") for element in all_datas]
            up_name = all_up_name[0].text
            writer.writerow([href_values[0], up_name])
            print(f'第{i}个视频已经爬取完成')
            i += 1

    ########################################################################################################################

    # 提取上一步爬取下来的文件中的url
    all_urls = pd.read_csv('./data/top100_url-游戏.csv',encoding='utf-8')
   # print(all_urls.columns)

    all_video_urls = all_urls['b站实时排行榜前一百视频url']
    all_video_up = all_urls['up主昵称']

    driver = webdriver.Chrome()
    csv_file = "data/top100_details-游戏.csv"

    with open(csv_file, 'a', newline='', encoding='utf-8') as f:
       # driver.find_element(By.LINK_TEXT,u"下一页").click()
        writer = csv.writer(f)
        writer.writerow(['视频标题', 'up主', '观看量', '弹幕数', '点赞数', '投币数', '收藏数'])

        i = 0
        for url in all_video_urls:
            driver.get(url)

            data_title = driver.find_elements(By.XPATH,f'//*[@id="viewbox_report"]/div/div/h1')
            ##33##3###3// *[ @ id = "viewbox_report"] / h1
            title = data_title[0].text  ###### 视频标题

            up = all_video_up[i]  ###### up主

            data_watch_dm = driver.find_elements(By.XPATH,'//*[@id="viewbox_report"]/div[2]/div/div/div')
#// *[ @ id = "viewbox_report"] / div / div / span
            watch = data_watch_dm[0].text  ###### 播放量

            if watch[-1] in '万':
                num = float(watch[0:-1])
                num *= 10000
                watch = str(num)

            dm = data_watch_dm[1].text  ###### 弹幕数
            if dm[-1] in '万':
                num = float(dm[0:-1])
                num *= 10000
                dm = str(num)

            data_dz_tb_sc_fx = driver.find_elements(By.XPATH,'//*[@id="arc_toolbar_report"]/div/div/div/div/span')


            video_like_info = data_dz_tb_sc_fx[0].text  ###### 点赞数
            if video_like_info[-1] in '万':
                num = float(video_like_info[0:-1])
                num *= 10000
                video_like_info = str(num)

            video_coin_info = data_dz_tb_sc_fx[1].text  ###### 投币数
            if video_coin_info[-1] in '万':
                num = float(video_coin_info[0:-1])
                num *= 10000
                video_coin_info = str(num)

            video_fav_info = data_dz_tb_sc_fx[2].text  ###### 收藏数
            if video_fav_info[-1] in '万':
                num = float(video_fav_info[0:-1])
                num *= 10000
                video_fav_info = str(num)

            #data_dz_tb_sc_fx1 = driver.find_elements(By.XPATH, '//*[@id="arc_toolbar_report"]/div/div/div[4]/div/span/div/div/span')
           # video_share_info = data_dz_tb_sc_fx1[0].text  ###### 分享数
           # if video_share_info[-1] in '万':
              #  num = float(video_share_info[0:-1])
               # num *= 10000
               # video_share_info = str(num)
               # print(video_share_info)

            row = [title, up, watch, dm, video_like_info, video_coin_info,
                   video_fav_info]
            #video_share_info
            writer.writerow(row)
            print(f'第{i + 1}个视频已经爬取成功！')
            i += 1