Python多线程爬虫

Python多线程爬虫

需求教程(旧版)：bilibili传送门
完整代码

import requests
from lxml import etree
import re
import os
from multiprocessing.dummy import Pool
# 保存视频
def get_video(video_all):
    print(video_all['title'],'正在下载...')
    headers = {
        'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36'
    }
    response = requests.get(video_all['true_url'],headers = headers)
    mp4 = response.content
    code = response.status_code
    with open('.\pearvideo\%s.mp4'%video_all['title'],'wb') as fp:
        fp.write(mp4)
        print(video_all['title'],'响应码:',code,end=" ")
        if code == 200:
            print('视频保存成功')
        else:
            print('视频保存失败')
# 创建文件夹
if not os.path.exists("./pearvideo"):
    os.mkdir("./pearvideo")
# 获取主页面
url = 'https://www.pearvideo.com/category_5'
headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36'
}
text = requests.get(url,headers = headers).text
tree = etree.HTML(text)
li_list = tree.xpath('//*[@id="listvideoListUl"]/li/div/a/@href')
print(li_list)
video_list = []
for li in li_list:
    # 在XHR中找到json()数据，其中包含mp4的url
    ex = 'video_([\d]*)'
    v_li = re.findall(ex,li)[0]
    v_url = 'https://www.pearvideo.com/videoStatus.jsp'
    v_headers = {
        'Referer':'https://www.pearvideo.com/'+li,
        'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36'
    }
    v_param = {
        'contId':v_li
    }
    v_text = requests.get(v_url,params = v_param,headers = v_headers).json()
    mp4_url = v_text['videoInfo']['videos']['srcUrl']
    '''
    mp4_url
    https://video.pearvideo.com/mp4/adshort/20210201/1612171096323-15594744_adpkg-ad_hd.mp4
    true_url
    'https://video.pearvideo.com/mp4/adshort/20210201/cont-1718799-15594744_adpkg-ad_hd.mp4
    '''
    mp4_x = 'https://video.pearvideo.com/mp4/adshort/([\d]*?)/[\d]*?-([\d]*?)_'
    mp4_re = re.findall(mp4_x,mp4_url)[0]
    mp4_date = mp4_re[0]
    mp4_id = mp4_re[1]
    true_url = 'https://video.pearvideo.com/mp4/adshort/'+mp4_date+'/cont-'+v_li+'-'+mp4_id+'_adpkg-ad_hd.mp4'
    print(true_url)
    # 获取标题
    t_url = 'https://www.pearvideo.com/'+li
    t_text = requests.get(t_url,headers = headers).text
    t_tree = etree.HTML(t_text)
    tx = '(.*?)_'
    title = re.findall(tx,t_tree.xpath('//title/text()')[0])[0]
    print(title)
    video_all = {
        'true_url':true_url,
        'title':title
    }
    video_list.append(video_all)
print()
print(video_list)
print()
# 创建线程池，调用下载函数
pool = Pool(4)
pool.map(get_video,video_list)
# 关闭线程池
pool.close()
pool.join()