爬虫项目——B站石原里美视频数据爬取

1. 爬虫原理

通过网页的源代码锁定目标数据区域，利用已有库函数取出目标数据。所需知识涉及 Python 、Java Script 等。

2. 数据爬取

2.1 库函数的引用

import jieba
import numpy
import wordcloud
import random
import requests
from time import sleep
from lxml import etree
from PIL import Image  # 加载评论的这些词汇

2.2 定义URL翻页函数

观察数个url寻找规律，一般是某一层数字的线性叠加。以本项目为例，url的规律是在https://search.bilibili.com/all?keyword=%E7%9F%B3%E5%8E%9F%E9%87%8C%E7%BE%8E后面添加&page=1,2,3等表示页数。

# 定义一个更新url的函数来取得翻页的效果
def update_url(base_url, page):
    url=base_url+"&page="+str(page)                            #经过观察页面我们发现，b站中下一页的链接并不再按钮中，但是下一页的url仅仅是在开始
    return url                                                 #页面的url后增加了“page=”页面数的操作，所以增加一个page变量可以做到效果，并方便控制爬取页数

2.3 定义储存TXT文件函数

# 定义一个储存函数来将取回来的数据存储为txt文件,名字是“data.txt”
def save_content(content):
    with open("data.txt", "a", encoding="utf-8") as writer:
        writer.write(content)
        writer.write("\n")
        writer.flush()

2.4 定义爬取函数

# 定义一个函数来封装爬取过程，每次调用此函数将爬取一个页面所有视频的相关信息
def spyder(url):
    r=requests.get(url, headers=headers)                        #函数中注释掉的内容是在将数据存储成csv文件时需要用到的
    html=etree.HTML(r.content.decode())
    # mixinlist=html.xpath('//div[@class="mixin-list"]')[0]
    # L=[]                                                      #此语句定义的L用于把数据储存成csv形式，用于txt储存这要注释掉
    videolist=html.xpath('.//ul[@class="video-list clearfix"]')

    for li in videolist:

        vm=li.xpath('.//li[@class="video-item matrix"]')
        for k in vm:                                           #用for循环遍历来储存页面中所有视频的信息
            # l=[]
            title=k.xpath('.//a[@class="img-anchor"]')[0]      #获取标签中title的值并输出
            print(title.attrib.get('title'))
            save_content(title.attrib.get('title'))            #调用储存函数储存title
            # l.append(title.attrib.get('title'))

            herf=title.attrib.get('href')                      #获取视频的链接
            print(herf)
            save_content(herf)                                 #调用储存函数储存链接
            # l.append(herf)

            img=title.xpath('.//div[@class="img"]')[0]
            span=img.xpath('.//span[@class="so-imgTag_rb"]')[0]
            time=span.text                                     #获取视频的视频长度
            # print(type(title.attrib.get('title')))
            print(time)
            # l.append(time)
            save_content(time)                                 #保存视频长度

            info = k.xpath('.//div[@class="info"]')[0]
            tags=info.xpath('.//div[@class="tags"]')[0]
            watch_num=tags.xpath('.//span[@class="so-icon watch-num"]/text()')[0]
            watch_num=watch_num.strip()                        #获取视频的观看次数
            print(watch_num)
            # # l.append(watch_num)
            # L.append(l)
            save_content(watch_num)                            #保存观看次数

2.5 主函数

• 利用User-Agent用户代理的方式对所需爬取的网站提供所使用的浏览器类型及版本、操作系统及版本、浏览器内核、等信息的标识，以达到防爬虫的目的。如果需要的访问次数量过大，一个用户代理也可能被识别为爬虫，建议使用用户代理池并在每次访问时候随机选择。
• 找到本浏览器User-Agent的方法：在开发者选项中选择Network，Name中随机点一个，Headers的尾部即可找到。

   ##################################################爬取数据##################################################################
    url = "https://search.bilibili.com/all?keyword=%E7%9F%B3%E5%8E%9F%E9%87%8C%E7%BE%8E"
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36"}
    page = 1
    base_url = url
    L1 = []  # 此L1列表是将数据储存成csv形式所需要用到的
    while url:
        spyder(url)
        # L1.append(L)                             #此语句也是将数据储存成csv形式所需要用到的
        page = page + 1
        if page == 11:  # 我们限制爬取10页的数据，当10页爬完后break语句跳出循环
            break
        url = update_url(base_url, page)
        sleep(random.random() * 10)

2.6 运行结果

3. 数据可视化

3.1 库函数的应用

import matplotlib.pyplot as plt
plt.rcParams['font.sans-serif'] = ['SimHei']  # 用来正常显示中文标签

3.2 定义写入TXT文件函数

# 分类数据写入txt文件
def SaveTxt(data_txt, TxtName, n):
    data_txt = str(data_txt)
    with open(TxtName, "a", encoding='UTF-8') as writer:
        writer.write(str(int(n / 4) + 1) + ":")
        for blog in data_txt:
            writer.write(blog)
            writer.flush()
        writer.write("\n")

3.3 定义读取TXT文件函数

# 读取txt文件
def ReadTxt(TxtName):
    with open(TxtName, "r", encoding="utf-8") as txts:
        word_txt = txts.read()
    with open(TxtName, "r", encoding="utf-8") as txts:
        data_txts = txts.readlines()
    return data_txts, word_txt

3.4 定义时间转换函数

# 时间转换为妙
def Second(x):
    h, m, s = x.strip().split(':')  # .split()函数将其通过':'分隔开，.strip()函数用来除去空格
    return int(h) * 3600 + int(m) * 60 + int(s)  # int()函数转换成整数运算

3.5 数据预处理

# 数据预处理
def Pretreatment(n):
    global sec, pre # 全局作用域
    # 地址：删除//
    data_txt[n + 1] = data_txt[n + 1][2:]
    # 时间：换成秒
    if len(data_txt[n + 2]) < 6:
        data_txt[n + 2] = "00:" + data_txt[n + 2]
        sec = Second(data_txt[n + 2])
    # 播放量：删除最后一个字
    if data_txt[n + 3].endswith('万'):
        data_txt[n + 3] = data_txt[n + 3][:-1]
        pre = float(data_txt[n + 3])
    else:
        pre = float(data_txt[n + 3]) / 100000
    return sec, pre


# 播放量与时长的关系
def Relationship(number):
    global XSclicks, Sclicks, Mclicks, Lclicks, XLclicks

    if number.time < 60:
        XSclicks += number.broadcast
    elif number.time < 120:
        Sclicks += number.broadcast
    elif number.time < 180:
        Mclicks += number.broadcast
    elif number.time < 240:
        Lclicks += number.broadcast
    else:
        XLclicks += number.broadcast
    return XSclicks, Sclicks, Mclicks, Lclicks, XLclicks

3.6 可视化

# 数据可视化
def Data_Visualization(XSclicks, Sclicks, Mclicks, Lclicks, XLclicks, strs):
    # 词云
    words = jieba.lcut(strs)  # 分词
    words_string = ' '.join(words)  # 将词连成一个大字符串，中间使用空格隔开每个词
    img = Image.open("lashou.jpg")
    bg_image = numpy.array(img)  # 底层需要将图片转换成矩阵
    my_wc = wordcloud.WordCloud(  # 绘制词云  需要字体，字体需要支持中文   词云产生过程   先构建模板，然后输入内容填充
        font_path='STFANGSO.TTF',  # 字体
        background_color='white',  # 背景色
        mask=bg_image  # 参考图片，模板图片
    ).generate(words_string)  # 填充词
    my_wc.to_file("sylm.png") # 存储词云图片


    # 饼状图
    name_list = ['小于60s的播放量：' + str(XSclicks) + '万',
                 '介于60s-120s的播放量:' + str(Sclicks) + '万',
                 '介于120s-180s的播放量:' + str(Mclicks) + '万',
                 '介于180s-240s的播放量:' + str(Lclicks) + '万',
                 '超过240s的播放量:' + str(XLclicks) + '万']
    num_list = [XSclicks, Sclicks, Mclicks, Lclicks, XLclicks]
    colors = ['green', 'yellow', 'blue', 'red', 'purple']
    plt.figure(1, figsize=(7, 7))  # 圆形
    expl = [0, 0, 0, 0, 0.1]  # 决定分割部分，及其与其它部分之间的间距
    plt.pie(x = num_list, explode=expl, labels=name_list, autopct='%3.1f %%', colors=colors, shadow=True)
    plt.title("不同时长的播放量(次)")
    plt.show()

3.9 定义类函数

# 定义类
class Vedio(object):
    def __init__(self, title, address, time, broadcast):
        self.title = title
        self.address = address
        self.time = time
        self.broadcast = broadcast

3.8 主函数

    ##################################################处理数据##################################################################
    # 初始化
    n = 0
    XSclicks = 0
    Sclicks = 0
    Mclicks = 0
    Lclicks = 0
    XLclicks = 0


    # 读取txt数据
    data_txts, word_txts = ReadTxt("data.txt")
    data_txt = [x.strip() for x in data_txts if x.strip() != '']  # 删除回车
    num = int(len(data_txt) / 4)


    # 数据分类并导出
    while n < num:
        secs, pres = Pretreatment(n)
        number = Vedio(data_txt[n], data_txt[n + 1], secs, pres)
        XSclicks, Sclicks, Mclicks, Lclicks, XLclicks =Relationship(number)
        SaveTxt(number.title, "title.txt", n)
        SaveTxt(number.address, "address.txt", n)
        SaveTxt(number.time, "time.txt", n)
        SaveTxt(number.broadcast, "broadcast.txt", n)
        n += 4


    # 可视化
    Data_Visualization(XSclicks, Sclicks, Mclicks, Lclicks, XLclicks, word_txts)

3.9 运行结果

1. 词云
2. 饼状图

4.全部代码

import jieba
import numpy
import wordcloud
import random
import requests
from time import sleep
from lxml import etree
from PIL import Image  # 加载评论的这些词汇

import matplotlib.pyplot as plt
plt.rcParams['font.sans-serif'] = ['SimHei']  # 用来正常显示中文标签


# 定义一个储存函数来将取回来的数据存储为txt文件,名字是“data.txt”
def save_content(content):
    with open("data.txt", "a", encoding="utf-8") as writer:
        writer.write(content)
        writer.write("\n")
        writer.flush()


# 定义一个函数来封装爬取过程，每次调用此函数将爬取一个页面所有视频的相关信息
def spyder(url):
    r=requests.get(url, headers=headers)                        #函数中注释掉的内容是在将数据存储成csv文件时需要用到的
    html=etree.HTML(r.content.decode())
    # mixinlist=html.xpath('//div[@class="mixin-list"]')[0]
    # L=[]                                                      #此语句定义的L用于把数据储存成csv形式，用于txt储存这要注释掉
    videolist=html.xpath('.//ul[@class="video-list clearfix"]')

    for li in videolist:

        vm=li.xpath('.//li[@class="video-item matrix"]')
        for k in vm:                                           #用for循环遍历来储存页面中所有视频的信息
            # l=[]
            title=k.xpath('.//a[@class="img-anchor"]')[0]      #获取标签中title的值并输出
            print(title.attrib.get('title'))
            save_content(title.attrib.get('title'))            #调用储存函数储存title
            # l.append(title.attrib.get('title'))

            herf=title.attrib.get('href')                      #获取视频的链接
            print(herf)
            save_content(herf)                                 #调用储存函数储存链接
            # l.append(herf)

            img=title.xpath('.//div[@class="img"]')[0]
            span=img.xpath('.//span[@class="so-imgTag_rb"]')[0]
            time=span.text                                     #获取视频的视频长度
            # print(type(title.attrib.get('title')))
            print(time)
            # l.append(time)
            save_content(time)                                 #保存视频长度

            info = k.xpath('.//div[@class="info"]')[0]
            tags=info.xpath('.//div[@class="tags"]')[0]
            watch_num=tags.xpath('.//span[@class="so-icon watch-num"]/text()')[0]
            watch_num=watch_num.strip()                        #获取视频的观看次数
            print(watch_num)
            # # l.append(watch_num)
            # L.append(l)
            save_content(watch_num)                            #保存观看次数



    # return L


            # info = k.xpath('.//div[@class="info"]')[0]
            # hide = info.xpath('.//div[@class="des hide"]')[0]
            # text = hide.text
            # print(text)
            # save_content(text)


# 定义一个更新url的函数来取得翻页的效果
def update_url(base_url, page):
    url=base_url+"&page="+str(page)                            #经过观察页面我们发现，b站中下一页的链接并不再按钮中，但是下一页的url仅仅是在开始
    return url                                                 #页面的url后增加了“page=”页面数的操作，所以增加一个page变量可以做到效果，并方便控制爬取页数


# 分类数据写入txt文件
def SaveTxt(data_txt, TxtName, n):
    data_txt = str(data_txt)
    with open(TxtName, "a", encoding='UTF-8') as writer:
        writer.write(str(int(n / 4) + 1) + ":")
        for blog in data_txt:
            writer.write(blog)
            writer.flush()
        writer.write("\n")


# 读取txt文件
def ReadTxt(TxtName):
    with open(TxtName, "r", encoding="utf-8") as txts:
        word_txt = txts.read()
    with open(TxtName, "r", encoding="utf-8") as txts:
        data_txts = txts.readlines()
    return data_txts, word_txt


# 定义类
class Vedio(object):
    def __init__(self, title, address, time, broadcast):
        self.title = title
        self.address = address
        self.time = time
        self.broadcast = broadcast


# 时间转换为妙
def Second(x):
    h, m, s = x.strip().split(':')  # .split()函数将其通过':'分隔开，.strip()函数用来除去空格
    return int(h) * 3600 + int(m) * 60 + int(s)  # int()函数转换成整数运算


# 数据预处理
def Pretreatment(n):
    global sec, pre # 全局作用域
    # 地址：删除//
    data_txt[n + 1] = data_txt[n + 1][2:]
    # 时间：换成秒
    if len(data_txt[n + 2]) < 6:
        data_txt[n + 2] = "00:" + data_txt[n + 2]
        sec = Second(data_txt[n + 2])
    # 播放量：删除最后一个字
    if data_txt[n + 3].endswith('万'):
        data_txt[n + 3] = data_txt[n + 3][:-1]
        pre = float(data_txt[n + 3])
    else:
        pre = float(data_txt[n + 3]) / 100000
    return sec, pre


# 播放量与时长的关系
def Relationship(number):
    global XSclicks, Sclicks, Mclicks, Lclicks, XLclicks

    if number.time < 60:
        XSclicks += number.broadcast
    elif number.time < 120:
        Sclicks += number.broadcast
    elif number.time < 180:
        Mclicks += number.broadcast
    elif number.time < 240:
        Lclicks += number.broadcast
    else:
        XLclicks += number.broadcast
    return XSclicks, Sclicks, Mclicks, Lclicks, XLclicks


# 数据可视化
def Data_Visualization(XSclicks, Sclicks, Mclicks, Lclicks, XLclicks, strs):
    # 词云
    words = jieba.lcut(strs)  # 分词
    words_string = ' '.join(words)  # 将词连成一个大字符串，中间使用空格隔开每个词
    img = Image.open("lashou.jpg")
    bg_image = numpy.array(img)  # 底层需要将图片转换成矩阵
    my_wc = wordcloud.WordCloud(  # 绘制词云  需要字体，字体需要支持中文   词云产生过程   先构建模板，然后输入内容填充
        font_path='STFANGSO.TTF',  # 字体
        background_color='white',  # 背景色
        mask=bg_image  # 参考图片，模板图片
    ).generate(words_string)  # 填充词
    my_wc.to_file("sylm.png") # 存储词云图片


    # 饼状图
    name_list = ['小于60s的播放量：' + str(XSclicks) + '万',
                 '介于60s-120s的播放量:' + str(Sclicks) + '万',
                 '介于120s-180s的播放量:' + str(Mclicks) + '万',
                 '介于180s-240s的播放量:' + str(Lclicks) + '万',
                 '超过240s的播放量:' + str(XLclicks) + '万']
    num_list = [XSclicks, Sclicks, Mclicks, Lclicks, XLclicks]
    colors = ['green', 'yellow', 'blue', 'red', 'purple']
    plt.figure(1, figsize=(7, 7))  # 圆形
    expl = [0, 0, 0, 0, 0.1]  # 决定分割部分，及其与其它部分之间的间距
    plt.pie(x = num_list, explode=expl, labels=name_list, autopct='%3.1f %%', colors=colors, shadow=True)
    plt.title("不同时长的播放量(次)")
    plt.show()


# _____________main_____________
if __name__ == '__main__':
    ##################################################爬取数据##################################################################
    url = "https://search.bilibili.com/all?keyword=%E7%9F%B3%E5%8E%9F%E9%87%8C%E7%BE%8E"
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36"}
    page = 1
    base_url = url
    L1 = []  # 此L1列表是将数据储存成csv形式所需要用到的
    while url:
        spyder(url)
        # L1.append(L)                             #此语句也是将数据储存成csv形式所需要用到的
        page = page + 1
        if page == 11:  # 我们限制爬取10页的数据，当10页爬完后break语句跳出循环
            break
        url = update_url(base_url, page)
        sleep(random.random() * 10)


    ##################################################处理数据##################################################################
    # 初始化
    n = 0
    XSclicks = 0
    Sclicks = 0
    Mclicks = 0
    Lclicks = 0
    XLclicks = 0


    # 读取txt数据
    data_txts, word_txts = ReadTxt("data.txt")
    data_txt = [x.strip() for x in data_txts if x.strip() != '']  # 删除回车
    num = int(len(data_txt) / 4)


    # 数据分类并导出
    while n < num:
        secs, pres = Pretreatment(n)
        number = Vedio(data_txt[n], data_txt[n + 1], secs, pres)
        XSclicks, Sclicks, Mclicks, Lclicks, XLclicks =Relationship(number)
        SaveTxt(number.title, "title.txt", n)
        SaveTxt(number.address, "address.txt", n)
        SaveTxt(number.time, "time.txt", n)
        SaveTxt(number.broadcast, "broadcast.txt", n)
        n += 4


    # 可视化
    Data_Visualization(XSclicks, Sclicks, Mclicks, Lclicks, XLclicks, word_txts)