#-*- coding:utf-8 _*-
"""
@version:
author:json
@time: 2020/07/18
@function:
@modify:
"""
from pyquery import PyQuery as pq
import time
import asyncio
import requests
from aiohttp import ClientSession
headers = {
# "referer": "https://www.jdlingyu.com/collection/meizitu",
"user-agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36"
}
id = 0
res = requests.get("https://www.jdlingyu.com/mzitu/71853.html",headers=headers)
doc = pq(res.text)
img_list = doc.find("div.entry-content img").items()
img_link_list = []
for img_link in img_list:
img_link_list.append(img_link.attr("src"))
async def __download_img( content):
global id
id += 1
file_name = str(id)
with open(file_name + '.jpg', 'wb') as f:
f.write(content)
return '下载第%s张图片成功' % file_name
async def fetch_img(url, seamphore):
async with seamphore:
async with ClientSession() as session:
async with session.get(url, headers=headers) as response:
content = await response.read()
mess = await __download_img(content)
return mess
if __name__ == '__main__':
start = time.time()
# 创建一个事件循环
loop = asyncio.get_event_loop()
# 定义最大并发量
semaphore = asyncio.Semaphore(500)
# 创建携程任务
tasks = [asyncio.ensure_future(fetch_img(img_url, semaphore)) for img_url in img_link_list]
# 启动事件循环
result = loop.run_until_complete(asyncio.gather(*tasks))
print("result======>>>",result)
# 关闭事件循环
loop.close()
print("消耗事件===>>",time.time()-start)
python 协程 async asyncio关键字
最新推荐文章于 2024-03-24 10:12:54 发布