B站爬虫-国王排名短评爬取

最终效果如下

首先是抓包，这里略去,需要构造的是cursor部分，这个会随着text文本返回

https://api.bilibili.com/pgc/review/short/list?media_id=28235154&ps=20&sort=0&cursor=81351010247255https://api.bilibili.com/pgc/review/short/list?media_id=28235154&ps=20&sort=0&cursor=81351010247255

import requests
import json
import pandas as pd
import time

url='https://api.bilibili.com/pgc/review/short/list?media_id=28235154&ps=20&sort=0'

header={'Accept':"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
"Accept-Encoding":"gzip, deflate, br",
"Accept-Language":"zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2",
"Cache-Control":"max-age=0",
"Connection":"keep-alive",
"Cookie":"uuid=6BA657CD-15B0-28F6-5736-8B01932EA13B17085infoc; buvid3=FC9FCF7F-2D83-4ED2-86A5-240930021967167632infoc; innersign=0; CURRENT_BLACKGAP=1; CURRENT_FNVAL=80; CURRENT_QUALITY=0; sid=7lb4zalx; rpdid=|(u)~ll~)muY0J'uYJY|Rumu|",
"Host":"api.bilibili.com",
"Sec-Fetch-Dest":"document",
"Sec-Fetch-Mode":"navigate",
"Sec-Fetch-Site":"none",
"Sec-Fetch-User":"?1",
"TE":"trailers",
"Upgrade-Insecure-Requests":"1",
"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0"
}

定义抓取单页评分的函数

def get_item(url,data_frame,header):
    raw_data=requests.get(url,headers=header)
    data=json.loads(raw_data.text)['data']
    next=data['next']
    for item in data['list']:
        item_dict={}
        item_dict['用户名']=item['author']['uname']
        item_dict['打分']=item['score']
        item_dict['评论']=item['content']
        item_dict['点赞数']=item['stat']['likes']
        data_frame= data_frame.append(item_dict, ignore_index=True)
        # print(data_frame['用户名'])
    return data_frame,next,raw_data.status_code

定义抓取全部内容的函数

def get_all(url,page):
    data_frame=pd.DataFrame(columns=['用户名', '打分', '评论', '点赞数'])

    data_frame,next,sta=get_item(url,data_frame,hea)  ##首先在循环一次，获取构造下一次url所需内容
    i=1
    for page in range(page):
        nurl=url+'&cursor='+str(next)
        data_frame,next,sta=get_item(nurl,data_frame,hea)
        time.sleep(0.5)
        print('-----------正在采集第'+str(i)+'打分数据--------------')
        print(nurl)
        i=i+1
    return data_frame

爬取数据

result=get_all(url,300)

查看数据，写入文件保存

result

result.to_csv('pinglun.csv',index=False,encoding="utf-8-sig")