python爬取动态网页

分析

网站带有反扒，需要进行浏览器伪装，网站的地址需要进行拆包分析，用eval进行内容分析，一层一层剥取，直到找到视屏地址

#B站爬取
import requests

#获取网页内容
def GetNetPage(url,dt,header):
    content=requests.get(url,headers=header).text
    path="E:\restore"
    content_dic=eval(content)
    for item in content_dic["data"]["items"]:
        dowload_path=item["item"]["video_playurl"]
        dt[item["item"]["description"]]=dowload_path


def GetAllPage(header):
    dt={}
    for i in range(10):
        num=i*10+1
        url = "https://api.vc.bilibili.com/board/v1/ranking/top?page_size=10&next_offset="+str(num)+"&tag=%E4%BB%8A%E6%97%A5%E7%83%AD%E9%97%A8&platform=pc"
        GetNetPage(url,dt,header)
    return dt

header = {"Accept": "application/json, text/plain, */*", "Origin": "https://vc.bilibili.com","Referer": "https://vc.bilibili.com/p/eden/rank", "Sec-Fetch-Mode": "cors","User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36"}
dt=GetAllPage(header)
i=0
for item in dt:
   with open("E:\\restore\\%d.mp4"%i,"wb") as f:
       print("------Save:%d-----begin--------"%i)
       f.write(requests.get(dt[item],headers=header,stream=True).content)
       print("------Save:%d-----success------" %i)
   i+=1