python 多进程爬虫实践小项目

多进程：

from selenium.webdriver import Chrome
from selenium.webdriver import ChromeOptions
import time
import requests,os
from lxml import etree
import multiprocessing


'''
项目名称：Age动漫视频爬取器  
项目使用方法：于“自行配置参数区域”输入相应参数后直接运行
项目亮点：应用了多并发的多进程模块，在进行大量爬取的情况下能够更加有效地利用CPU资源，大大提高爬取效率：
          以 "火影忍者" 爬取24集 最大进程数4 为例 效率比单进程提高了近38% 
项目不足：该网站上有小部分视频如 "schooldays" 采用了blob协议加密，对于这样的视频暂时无法爬取   
目标网站：https://www.agedm.org/
作者：wzq(QQ:3217418192)
'''


def get_mp4_link(orgin_url,n):
    options = ChromeOptions()
    options.add_argument('--headless')
    options.add_argument('--disable-cpu')
    driver = Chrome(options=options)
    url = orgin_url.replace('detail', 'play') + f'/1/{n}'
    driver.get(url)
    time.sleep(2)
    driver.switch_to.frame('''iframeForVideo''')
    tree = etree.HTML(driver.page_source)
    return tree.xpath('''/html/body/div/div/video/@src''')[0]



def download_mp4(mp4_link,n,headers,father_path):
    print(f'开始下载第{n}集\n')
    start = time.time()
    mp4_data = requests.get(url=mp4_link, headers=headers).content
    time.sleep(1)
    with open(f"D:\code\爬虫练习\Age动漫视频捏\{father_path}\\{n}.mp4", 'wb') as fp:
        fp.write(mp4_data)
    end = time.time()
    print(f"第{n}集下载完成,花费时间{format(end - start)}秒钟\n")



def mian(orgin_url,n,headers,father_path):
    mp4_link=get_mp4_link(orgin_url,n)
    download_mp4(mp4_link,n,headers,father_path)

    
def main():
    print("开始计时捏")
    StartTime = time.time()

    ##··········自行配置参数区域····························
    ##···················································
    orgin_url = "https://www.agedm.org/detail/20020006"  # 输入起始url 例如 https://www.agedm.org/detail/20240086
    father_path = "火隐忍者1"  # 输入作品名称，用以命名文件夹
    limit = 24  # 输入一共爬取多少集
    max_processes = 4  # 输入最大进程数
    ##··········自行配置参数区域····························
    ##···················································


    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36"
    }


    try:
        os.mkdir(f"D:\code\爬虫练习\Age动漫视频捏\\{father_path}")
    except:
        print("文件夹已存在")

    with multiprocessing.Pool(processes=max_processes) as pool:
        for n in range(1, limit + 1):
            pool.apply_async(mian, args=(orgin_url, n,headers,father_path))
        pool.close()
        pool.join()
    EndTime=time.time()
    print(f"多进程程序共耗时{format(EndTime-StartTime)}秒")


if __name__ == '__main__':
    main()

运行耗时：

单进程：

from selenium.webdriver import Chrome#实例化对象
from selenium.webdriver import ChromeOptions#实例化对象
import time
import requests,os
from lxml import etree


options = ChromeOptions()
options.add_argument('--headless')
options.add_argument('--disable-cpu')
driver = Chrome(options=options)

orgin_url = "https://www.agedm.org/detail/20020006"
limit=24

headers={
    "User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36"
}

def download_mp4(mp4_link,n):
    mp4_data = requests.get(url=mp4_link, headers=headers).content
    time.sleep(1)
    with open(f"D:\code\爬虫练习\Age动漫视频捏\{father_path}\\{n}.mp4", 'wb') as fp:
        fp.write(mp4_data)


father_path="火影忍者2"
try :
    os.mkdir(f"D:\code\爬虫练习\Age动漫视频捏\\{father_path}")
except:
    print("文件夹已存在")

StartTime=time.time()
for n in range(1,limit+1):
    url=orgin_url.replace('detail','play')+f'/1/{n}'
    driver.get(url)
    time.sleep(2)
    driver.switch_to.frame('''iframeForVideo''')
    tree = etree.HTML(driver.page_source)
    mp4_link = tree.xpath('''/html/body/div/div/video/@src''')[0]

    print(f'开始下载第{n}集\n')
    start = time.time()
    download_mp4(mp4_link=mp4_link,n=n)
    end=time.time()
    print(f"第{n}集下载完成,花费时间{end-start}秒钟\n")

EndTime=time.time()
print(f"单进程程序共耗时{format(EndTime-StartTime)}秒")

运行耗时：