import asyncio
import aiohttp
class AsnycGrab(object):
def __init__(self, url_list, max_threads):
self.urls = url_list
self.max_threads = max_threads #协程数
async def get_results(self, url):
# 异步发送请求
async with aiohttp.ClientSession() as session:
async with session.get(url, timeout=30) as response:
assert response.status == 200
html = await response.read()
print(url)
return 1
async def handle_tasks(self, task_id, work_queue):
while not work_queue.empty():
current_url = await work_queue.get() #从队列中获取url
try:
task_status = await self.get_results(current_url)
except Exception as e:
print(e)
def eventloop(self):
q = asyncio.Queue() # 协程队列
[q.put_nowait(url) for url in self.urls] # 讲url全部放到队列
loop = asyncio.get_event_loop() # 创建事件循环
tasks = [self.handle_tasks(task_id, q, ) for task_id in range(self.max_threads)] # 创建self.max_threads个任务
loop.run_until_complete(asyncio.wait(tasks))
loop.close()
if __name__ == '__main__':
async_example = AsnycGrab(['http://127.0.0.1:5000' for i in range(100000)], 500) #创建100000个任务,每次异步提交500个
async_example.eventloop()
使用aiohttp实现高并发网络请求
最新推荐文章于 2024-08-14 08:15:00 发布