并发发送异步网络请求
"""
注意:此处是DEMO,按需根据实际业务去改
"""
import asyncio
import aiohttp
class AsnycGrab(object):
def __init__(self, url_list, max_threads):
self.urls = url_list
self.max_threads = max_threads
async def get_results(self, url):
async with aiohttp.ClientSession() as session:
async with session.get(url, timeout=30) as response:
assert response.status == 200
html = await response.read()
print(url)
return 1
async def handle_tasks(self, task_id, work_queue):
while not work_queue.empty():
current_url = await work_queue.get()
try:
task_status = await self.get_results(current_url)
except Exception as e:
print(e)
def eventloop(self):
q = asyncio.Queue()
[q.put_nowait(url) for url in self.urls]
loop = asyncio.get_event_loop()
tasks = [self.handle_tasks(task_id, q, ) for task_id in range(self.max_threads)]
loop.run_until_complete(asyncio.wait(tasks))
loop.close()
if __name__ == '__main__':
async_example = AsnycGrab(['http://192.168.210.206:5000/v1/product/get_products' for i in range(10000)], 500)
async_example.eventloop()