需求:根据视频关键字,爬取该关键字所搜出来的视频的弹幕。
例如设置关键字“996”,代码会爬取根据“996”搜索出来的视频的弹幕,可以设置爬取的页码,开始页码和结束页码,代码自动获取每个视频的cid,再根据cid通过api获取弹幕信息。
设置要爬取的页码。直接上代码:
# This is a sample Python script.
# 301-400
import json
import os.path
import os
import random
import threading
import time
from hashlib import sha1
import requests
import re
import jieba
import winsound
from fake_useragent import UserAgent
from threading import Thread
# Press Shift+F10 to execute it or replace it with your code.
# Press Double Shift to search everywhere for classes, files, tool windows, actions, and settings.
class BainduSpider:
def __init__(self):
self.headers = {
'Accept':"*/*",
"Accept-Language": "zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3",
'User-Agent': UserAgent().random
}
#开始页码
self.startPageNum = 1
#结束页码
self.finalPageNum = 10 #包括该页码
self.errPageNum = 0
self.keyWord = ""
self.allVideoDanMuInfo = {"keyWord":'', "data":[]}
self.danMuInfoList = []
self.error = True
def allListUrlSet(self, pageNum, keyWord):
if pageNum == 1:
return "https://search.bilibili.com/all?keyword=" + keyWord
else:
return "https://search.bilibili.com/all?keyword=" + keyWord + "&page=" + str(pageNum) + "&o=" + str((pageNum - 1)*30)
# 根据url获取页面代码
def requestText(self, url):
listUrl = url
list_html = requests.get(url=listUrl, headers=self.headers, timeout=30)
list_html.encoding = "utf-8"
list_html = list_html.text
# print("requestText根据url获取页面代码:", list_html)
return list_html
# 筛选出关键数据数组
def findKeyData(self, findText, htmlText):
videoUrlList = re.findall(findText, htmlText)
print("findKeyData筛选出关键数据数组")
return videoUrlList
# 获取单个视频页面数据
def getDnMu(self, videoUrl):
videoUrl = "https://" + videoUrl
# 请求页面html数据
htmlText = self.requestText(videoUrl)
# print("弹幕页面数据:", htmlText)
return htmlText
# 获取视频cid
def getCid(self,htmlText):
findText = 'window.__INITIAL_STATE__={(.*?)};'
cidList = re.findall(findText, htmlText)
cidList = "{" + cidList[0] + "}"
cidJson = json.loads(cidList)
print("视频名称:", cidJson['videoData']['title'])
print("cid信息:", cidJson['videoData']['cid'])
cid = cidJson['videoData']['cid']
bvid = cidJson['videoData']['bvid']
title = cidJson['videoData']['title']
videoInfo = {"cid":cid, "bvid":bvid, "title":title}
return videoInfo
# 根据cid获取弹幕信息
def getDanMu(self, cid):
url = "https://api.bilibili.com/x/v1/dm/list.so?oid=" + str(cid)
# 获取信息
danMuHTML = self.requestText(url)
findText = '<d p=".*?">(.*?)</d>'
danMuList = self.findKeyData(findText, danMuHTML)
print("弹幕数量:", len(danMuList))
return danMuList
# 关键字搜索
def keyWordSearch(self):
for i in range(self.startPageNum, self.finalPageNum+1):
self.errPageNum += 1
# 请求的url
requestUrl = self.allListUrlSet(i, self.keyWord)
# 请求到的页面代码数据
htmlText = self.requestText(requestUrl)
findText = '<li class="video-item matrix"><a href="//(.*?)" title='
# 筛选出当前页面的视频链接
videoUrlList = self.findKeyData(findText, htmlText)
videoIndex = 0
# 访问视频链接,获取每个视频的弹幕信息
for videoUrl in videoUrlList:
videoIndex = videoIndex + 1
print("当前为第:" + str(i) + "页/(共" + str(self.finalPageNum - self.startPageNum + 1) + "页);第:" + str(videoIndex) + "/(共" + str(len(videoUrlList)) + ")个视频")
oneVideoHtml = self.getDnMu(videoUrl)
# 获取cid
videoInfo = self.getCid(oneVideoHtml)
cid = videoInfo['cid']
# 获取弹幕信息
danMuList = self.getDanMu(cid)
self.danMuInfoList.extend(danMuList)
saveInfo = {"videoInfo":videoInfo, "danMu":danMuList}
self.allVideoDanMuInfo['data'].append(saveInfo)
# print("视频信息:", self.allVideoDanMuInfo)
sleepTime = random.randint(1, 10)
print("等待时间:", sleepTime)
time.sleep(sleepTime)
self.allVideoDanMuInfo['keyWord'] = self.keyWord
self.saveData(self.allVideoDanMuInfo, self.danMuInfoList)
# 数据存入文件
def saveData(self, videoInfo, danMuList):
print("。。。。。。存储信息。。。。")
file = open("E://弹幕数据/" + self.keyWord + "(" + str(self.startPageNum) + "-" + str(self.errPageNum) + ").txt", 'w', encoding='utf-8')
file.write(str(videoInfo))
file.close()
if self.error:
for danmu in danMuList:
with open("E://弹幕数据/" + self.keyWord + "(" + str(self.startPageNum) + "-" + str(self.errPageNum) + ")String.txt", mode='a', encoding='utf-8') as f:
f.write(danmu)
f.write('\n')
self.error = False
winsound.Beep(500, 2000)
# Press the green button in the gutter to run the script.
if __name__ == '__main__':
# 设置要爬取的视频关键字
keyWordList = ['零元购', '国足', '华为', '苹果', 'ios', '为爱冲锋']
#国足
try:
baind = BainduSpider()
for keyWord in keyWordList:
baind.keyWord = keyWord
baind.danMuInfoList = []
baind.allVideoDanMuInfo['data'] = []
baind.error = True
baind.errPageNum = 0
baind.keyWordSearch()
except:
baind.saveData(baind.allVideoDanMuInfo, baind.danMuInfoList)
print("结束")
# See PyCharm help at https://www.jetbrains.com/help/pycharm/
生成词云的代码大家可以自己写一下,下面是根据“996”、“躺平”、“美国”、“零元购”的关键字爬取的弹幕词云