import requests
import random
import os
import re
from lxml import etree
from multiprocessing.dummy import Pool
if not os.path.exists('视频2'):
os.mkdir('视频2')
urls = []
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36',
}
url = 'https://www.pearvideo.com/category_5'
page_text = requests.get(url=url, headers=headers).text
tree = etree.HTML(page_text)
li_list = tree.xpath('//ul[@id="listvideoListUl"]/li')
for li in li_list:
detail_url = 'https://www.pearvideo.com/'+li.xpath('./div/a/@href')[0]
name = li.xpath('./div/a/div[2]/text()')[0]+'.mp4'
detail_page_text = requests.get(url=detail_url, headers=headers).text
contId = re.findall("\d+", detail_url)[0]
new_url = 'https://www.pearvideo.com/videoStatus.jsp'
params = {
'contId': contId,
'mrd': str(random.random())
}
headers["Referer"] = detail_url
video_json = requests.get(url=new_url, headers=headers, params=params).json()
video_list = video_json['videoInfo']['videos']['srcUrl']
video_title = '/'.join(video_list.split('/')[0:-1])
video_footer = '-'.join(video_list.split('-')[1:])
dic = {
'name': name,
'url': video_title +"/cont-" + contId + '-' + video_footer
}
urls.append(dic)
def get_video_data(dic):
video_data = requests.get(url=dic['url'], headers=headers).content
with open('./视频2/' + dic['name'], "wb") as fp:
fp.write(video_data)
print(dic['name'] + '下载成功!!!')
pool = Pool(4)
pool.map(get_video_data, urls)
pool.close()
pool.join()
python爬虫
最新推荐文章于 2024-08-03 14:18:54 发布