【 哔哩哔哩动漫信息获取（只局限于信息收集），存储数据】

哔哩哔哩动漫信息获取（只局限于信息收集），存储数据到excel或者mysql

一、确定你所需要数据来源的地址

数据来源页面和地址如下：
数据来源页面

数据来源地址，需要通过浏览器的开发者工具也就是F12 来获取对应的请求地址：

一般来说，像这样返回的json格式的数据最好获取的。。。。。。

二、准备python的依赖包

一般来说用到的插件不会太多，像请求静态页面的请求依赖包：requests、BeautifulSoup，动态js加载页面的：selenium，本文抓取的数据是json格式，所以一般requests依赖包就完成可以使用
所用到的依赖包如下：
                          requests(请求返回数据)、numpy(创建空数组)
                           pymysql(数据入库)、xlwt(数据写入文件xls)

三、代码编写

1、请求获取数据

def ts_requests(baseurl):
    data = {
        "user-agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36"
    }

    result = requests.get(url=baseurl,headers=data)
    result.encoding = 'UTF-8'
    if result.status_code == 200:
        result_data = result.json()
        return result_data
    else:
        print("请求失败，状态码为："+result.status_code)

2、处理响应数据

def ts_data_deal(url_front,url_end,numgegin,numend):
    real_data = np.empty((0,10),str)
    j = 0
    for ppp in range(numgegin, numend):
        baseurl = url_front + str(ppp) + url_end
        ts_data = ts_requests(baseurl)['data']['list']
        for i in ts_data:
            for key, value in i.items():
                if type(value) == dict:
                    if key == 'badge_info':
                        data_badge = value['text']  # 观看需要条件
                    elif key == 'first_ep':
                        data_cover = value['cover']  # 图片链接
                        data_ep_id = value['ep_id']  # id
                    else:
                        pass
                elif type(value) == int:
                    pass
                else:
                    pass
            data_link = i['link']  # 地址链接
            data_order = i['order']  # 追番数量
            data_subtitle = i['subTitle']  # 副标题
            data_title = i['title']  # 主标题
            data_index_show = i['index_show']  # 集数
            if i['score'] =='':
                data_score = '暂无评分'
            else:
                data_score = i['score'] + '分'  # 评分
            j+=1
            real_data = np.append(real_data,np.array([[j,data_badge,data_title,data_subtitle,data_cover,data_index_show,data_score,data_order,data_link,data_ep_id]]),axis=0)

    return real_data

3、数据入库

def ts_pymysql(datalist):
    conn = pymysql.connect(
        host='地址',
        port='端口',
        user='用户',
        password='密码',
        db='数据库'
    )
    cur = conn.cursor()

    sql = """drop table if exists table_name"""
    print('正在删除对应的表以及相关的数据，请稍等5秒钟.......')
    time.sleep(2)
    cur.execute(sql)
    conn.commit()
    print('删除成功！！！')

    sql_create = """CREATE TABLE table_name (
                        id INT PRIMARY KEY NOT NULL,
                        data_badge VARCHAR ( 255 ),
                        data_title VARCHAR ( 255 ),
                        data_subtitle VARCHAR ( 255 ),
                        data_index_show VARCHAR ( 255 ),
                        data_order VARCHAR ( 255 ),
                        data_score VARCHAR ( 255 ),
                        data_link VARCHAR ( 255 ),
                        data_cover VARCHAR(225),
                        data_ep_id  VARCHAR(225))
                        """
    try:
        cur.execute(sql_create)
        print("table_name创建表成功")
        sql_insert = """insert into table_name (id,data_badge,data_title,data_subtitle,data_index_show,data_order,data_score,data_link,data_cover,data_ep_id) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"""
        print('正在导入数据，请稍等5秒钟。。。。')
        for item in datalist:
            cur.execute(sql_insert,(int(item[0]),item[1],item[2],item[3],item[4],item[5],item[6],item[7],item[8],item[9]))
            conn.commit()
        print('导入成功，请查看')
    except Exception as e:
        print(e)

4、存储为表格excel

def tx_excel(datalist):
    workBook = xlwt.Workbook('UTF-8')
    print('正在检查文件是否存在。。。')
    sheet =workBook.add_sheet('bilibili_动漫_数据')
    i = 0
    print('正在写入操作，请等待一会')
    time.sleep(5)
    for num in datalist:
        for j in range(0,len(num)):
            sheet.write(i,j,num[j])
        i+=1
    workBook.save('./bilibili_动漫_数据.xls')
    print('写入完成')

                  这里的目录或者文件检查等，你可以os模块进行判断

path=“path\\”
if not os.path.isdir(path):
	os.mkdir(path)

4、其他汇总

访问：

def main():
    #动漫番剧地址
    url_front = "https://api.bilibili.com/pgc/season/index/result?st=1&order=3&season_version=-1&spoken_language_type=-1&area=-1&is_finish=-1&copyright=-1&season_status=-1&season_month=-1&year=-1&style_id=-1&sort=0&page="
    url_end = "&season_type=1&pagesize=20&type=1"
    #国创地址
    #url_front1 = "https://api.bilibili.com/pgc/season/index/result?st=4&order=3&season_version=-1&is_finish=-1&copyright=-1&season_status=-1&year=-1&style_id=-1&sort=0&page="
   # url_end1 = "&season_type=4&pagesize=20&type=1"

    num1 = int(input("请输入你要爬取页面数据的开始页："))
    num2 = int(input("请输入你要爬取页面数据的结束页："))
    ppppp = ts_data_deal(url_front, url_end,num1,num2+1)
    time.sleep(2)
    choose = int(input("请选择存储方式：1、mysql 2、excel 3、pic下载  请输入数字："))
    if choose == 1:
        ts_pymysql(ppppp)
    elif choose == 2:
        tx_excel(ppppp)
    elif choose ==3:
        tx_picture(ppppp)
    else:
        exit()



if __name__ == '__main__':
    main()