最终效果如下
首先是抓包,这里略去,需要构造的是cursor部分,这个会随着text文本返回
https://api.bilibili.com/pgc/review/short/list?media_id=28235154&ps=20&sort=0&cursor=81351010247255https://api.bilibili.com/pgc/review/short/list?media_id=28235154&ps=20&sort=0&cursor=81351010247255
import requests
import json
import pandas as pd
import time
url='https://api.bilibili.com/pgc/review/short/list?media_id=28235154&ps=20&sort=0'
header={'Accept':"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
"Accept-Encoding":"gzip, deflate, br",
"Accept-Language":"zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2",
"Cache-Control":"max-age=0",
"Connection":"keep-alive",
"Cookie":"uuid=6BA657CD-15B0-28F6-5736-8B01932EA13B17085infoc; buvid3=FC9FCF7F-2D83-4ED2-86A5-240930021967167632infoc; innersign=0; CURRENT_BLACKGAP=1; CURRENT_FNVAL=80; CURRENT_QUALITY=0; sid=7lb4zalx; rpdid=|(u)~ll~)muY0J'uYJY|Rumu|",
"Host":"api.bilibili.com",
"Sec-Fetch-Dest":"document",
"Sec-Fetch-Mode":"navigate",
"Sec-Fetch-Site":"none",
"Sec-Fetch-User":"?1",
"TE":"trailers",
"Upgrade-Insecure-Requests":"1",
"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0"
}
定义抓取单页评分的函数
def get_item(url,data_frame,header):
raw_data=requests.get(url,headers=header)
data=json.loads(raw_data.text)['data']
next=data['next']
for item in data['list']:
item_dict={}
item_dict['用户名']=item['author']['uname']
item_dict['打分']=item['score']
item_dict['评论']=item['content']
item_dict['点赞数']=item['stat']['likes']
data_frame= data_frame.append(item_dict, ignore_index=True)
# print(data_frame['用户名'])
return data_frame,next,raw_data.status_code
定义抓取全部内容的函数
def get_all(url,page):
data_frame=pd.DataFrame(columns=['用户名', '打分', '评论', '点赞数'])
data_frame,next,sta=get_item(url,data_frame,hea) ##首先在循环一次,获取构造下一次url所需内容
i=1
for page in range(page):
nurl=url+'&cursor='+str(next)
data_frame,next,sta=get_item(nurl,data_frame,hea)
time.sleep(0.5)
print('-----------正在采集第'+str(i)+'打分数据--------------')
print(nurl)
i=i+1
return data_frame
爬取数据
result=get_all(url,300)
查看数据,写入文件保存
result
result.to_csv('pinglun.csv',index=False,encoding="utf-8-sig")