import asyncio
from aiohttp import ClientSession, TCPConnector
asyncdefaiohttp_get():
url ='url'
conn = TCPConnector(limit=10)# 限制同时链接数,连接默认是100,limit=0 无限制asyncwith ClientSession(connector=conn)as session:asyncwith session.get(url)as response:
html =await response.text()return html
2、Semaphore 信号量
asyncdefasyncSpider(sem, url):"""异步任务"""asyncwith sem:print('Getting data on url',url)asyncwith ClientSession()as session:asyncwith session.get(url)as response:
html =await response.text()return html
asyncdeftaskManager():"""异步任务管理器"""
tasks =[]
sem = asyncio.Semaphore(10)# 控制并发数for url in url_list:
task = asyncio.create_task(asyncSpider(sem, url))
task.add_done_callback(parseHTML)
tasks.append(task)await asyncio.gather(*tasks)defmain():print('Task start! It is working...')
loop = asyncio.get_event_loop()
loop.run_until_complete(taskManager())print('Finished!')