异步多视频爬取
注意点:
1.多线程
2.js渲染html
import requests
import re
from lxml import etree
from multiprocessing.dummy import Pool
import os
if not os.path.exists("video"):
os.mkdir("video")
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36"
}
url='https://www.pearvideo.com'
#存放视频名字和数据链接
video_all=[]
res=requests.get(url=url,headers=headers).text
tree=etree.HTML(res)
video_list=tree.xpath('//ul[@id="actRecommendCont"]/li/div')
for li in video_list:
#拼接视频名称
video_name=li.xpath('./a/h3/text()')[0]+'.mp4'
#拼接视频播放界面
video_url="https://www.pearvideo.com/"+li.xpath('./a/@href')[0]
#请求视频播放界面
res2=requests.get(url=video_url,headers=headers).text
#由于视频链接是由js生成的,所以只能用re匹配
video_src=re.findall('srcUrl="(.*?)",vdo',res2)[0]
dic = {"name":video_name,"url":video_src}
video_all.append(dic)
#下载方法
def dow_video(vie_dic):
print(vie_dic['name'] + "下载中...")
res=requests.get(url=vie_dic['url'],headers=headers).content
with open("video/"+vie_dic['name'],'wb')as fw:
fw.write(res)
print(vie_dic['name']+"下载成功!")
#开启8个线程
pool=Pool(8)
#多线程执行
pool.map(dow_video,video_all)
#关闭pool,使其不在接受新的(主进程)任务
pool.close()
#主进程阻塞后,让子进程继续运行完成,子进程运行完后,再把主进程全部关掉。
pool.join()