python爬虫：检索B站视频提取弹幕构建词云

1、思路

B站的视频弹幕由三层关键词可以锁定，第一层是检索的关键词，第二层是每个视频的bvid，第三曾是每个视频的oid，相关的获取API网上都有。
爬虫使用的是request，构建词云使用的wordCloud，中间还嵌套使用了正则、数据分析部分内容。
总体的思路是：

1. 根据关键词检索视频
2. 根据每个视频的标示bvid得到标示该视频弹幕的oid
3. 根据oid请求弹幕，并解析弹幕数据
4. 根据获取的弹幕数据构建词云

2、源码

#!/usr/bin/env python
# -*- coding: utf-8 -*-

import matplotlib.pyplot as plt
from wordcloud import WordCloud, STOPWORDS
import requests
import json
import chardet
import re

# <a href="//www.bilibili.com/video/BV1o64y1u77p?from=search" title="
class GenWordCloud:
    def __init__(self,filename):
        self.filename=filename +'.txt'
        print('filename=',self.filename)

    def SearchVideo(self):
        search_name = input('您想要爬取的视频关键字是？\n(输入完毕请按回车)：')
        pages = 2# 设置爬取的总页数
        for page in range(1, pages):  # 翻页循环
            url = ('https://search.bilibili.com/all?keyword=' + search_name + '&single_column=0&&order=dm&page=' + str(
                page))  # 翻页循环设定
            print(url)
            r = requests.get(url)  # GET请求访问网页
            content = r.text  # 解析网页源码
            # print(content)
            pattern = re.compile('<a href="//www.bilibili.com/video/(.*?)\?from=search" title="')
            links = pattern.findall(content)  # 使用正则表达式从源码中找到所有视频地址
            # print(links)
            return links

    # 根据bvid请求得到oid
    def get_oid(self,vid):
        url = 'https://api.bilibili.com/x/player/pagelist?bvid=' + vid + '&jsonp=jsonp'
        res = requests.get(url).text
        json_dict = json.loads(res)
        return json_dict["data"][0]["cid"]

    # 根据oid请求弹幕，解析弹幕得到最终的数据
    def get_data(self,oid):
        url = "https://api.bilibili.com/x/v1/dm/list.so?oid=" + str(oid)
        # 打开网页
        res = requests.get(url)
        res.encoding = chardet.detect(res.content)['encoding']
        # 获取所有的内容
        final_res = res.text
        # 根据正则表达式提取出弹幕内容
        pattern = re.compile('<d.*?>(.*?)</d>')
        data = pattern.findall(final_res)
        return data

    #保存
    def save(self,data):
        #追加
        with open(self.filename, mode="a", encoding="utf-8") as file:
            for i in data:
                file.write(i)
                file.write("\n")

    def draw(self):
        # 打开文件
        path=r'./'+self.filename
        print('path=',path)
        with open(path, 'r', encoding='utf-8') as f:
            text = f.read()

        # 设置停用词
        stopwords = set(STOPWORDS)

        # 设置词云样式
        font = r'/System/Library/fonts/PingFang.ttc'
        output = WordCloud(font_path=font, background_color="black", max_words=2000,
                           stopwords=stopwords, max_font_size=40, random_state=42)

        # 生成词云
        output.generate(text)

        # 输出显示
        plt.figure(figsize=(8, 9))
        plt.imshow(output, interpolation="bilinear")
        # on or off
        plt.axis("off")
        plt.show()

if __name__ =="__main__":
    gwc=GenWordCloud("1")
    params=gwc.SearchVideo()
    print(params)
    # params=["BV1Az4y1Q7ru","BV1jC4y1b7bh","BV1jC4y1b7bh"]
    for i in params:
        oid = gwc.get_oid(i)
        data =gwc. get_data(oid)
        gwc.save(data)

    gwc.draw()

3、效果展示