bilibili弹幕爬虫， 2019-1-10

bilibili弹幕爬虫

示例网址：
https://www.bilibili.com/video/av21483276

视频弹幕网址：
https://comment.bilibili.com/35358033.xml

视频下载地址：
https://upos-hz-mirrorwcsu.acgvideo.com/upgcxcode/33/80/35358033/35358033-1-32.flv?e=ig8euxZM2rNcNbR1hwdVhoM1hWdVhwdEto8g5X10ugNcXBlqNxHxNEVE5XREto8KqJZHUa6m5J0SqE85tZvEuENvNC8xNEVE9EKE9IMvXBvE2ENvNCImNEVEK9GVqJIwqa80WXIekXRE9IMvXBvEuENvNCImNEVEua6m2jIxux0CkF6s2JZv5x0DQJZY2F8SkXKE9IB5QK==&deadline=1547186777&gen=playurl&nbs=1&oi=3056465252&os=wcsu&platform=pc&trid=2ee0f4513d654dc4a6b4cd9d37f2a59a&uipk=5&upsig=b5084f0c9b0308489c7836a3efee6fd9

其实是根据网上的帖子反推的

1. 找35358033
   
   在window.__playinfo__标签下

有点复杂，再找

找到了最右面视频的地址，直接可以下载

找了一圈
决定从下面入手

re.findall(r"\"pages\"\:\[\{\"cid\":(.*?)\,", res, re.S)[0]

上代码

import requests
from lxml import etree
import re
from pprint import pprint
import json


class Bili(object):
    def __init__(self, url_num):
        self.name = "av号" + str(url_num)
        self.url = "https://www.bilibili.com/video/av{}/".format(url_num)
        self.headers = {
            "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36"
        }

    def parse(self,url):
        response = requests.get(url, headers=self.headers)
        res = response.content.decode()
        return res

    def get_content(self, res):
        html = etree.HTML(res)
        item = {}
        item["title"] = html.xpath("//div[@id='viewbox_report']/h1/@title")[0] if len(
            html.xpath("//div[@id='viewbox_report']/h1/@title")) > 0 else None
        if item["title"]:
            item["cid"] = re.findall(r"\"pages\"\:\[\{\"cid\":(.*?)\,", res, re.S)[0]
        else:
            return None
        print(item)
        return item

    def get_url(self, item):
        cid = item["cid"]
        danmu_url = "https://comment.bilibili.com/{}.xml".format(cid)
        print(danmu_url)
        return danmu_url

    def get_danmu(self, res, item):
        print("0"*100)
        pprint(res)
        print("0" * 100)
        html = etree.HTML(res.encode())
        print(html)
        item["弹幕"] = html.xpath("//d/text()")
        # pprint(item)
        return item

    def save(self, name, content):
        with open("{}.json".format(name), "a", encoding="utf-8")as f:
            f.write(json.dumps(content, ensure_ascii=False, indent=4))
            print("保存成功")

    def run(self):
        item = {}
        # 1 获取url
        # 2 发送请求，获取相应
        res = self.parse(self.url)
        # 3 提取cid和标题
        item = self.get_content(res)
        if item == None:
            print("vid号不正确，请重新输入。")
            return
        # 4 组合弹幕url
        danmu_url = self.get_url(item)
        # 5 发送请求获取相应
        res_danmu = self.parse(danmu_url)
        # 6 提取
        end = self.get_danmu(res_danmu, item)
        # 7 保存
        self.save(self.name, end)
        print("程序结束")


if __name__ == '__main__':
    # url_num = input("请输入8位视频av号:")
    url_num = 21483276
    print("url_num=", url_num)
    b = Bili(url_num)
    b.run()

参考

https://blog.csdn.net/qq_22043649/article/details/80985903

https://blog.csdn.net/a360316515/article/details/83058481