asyncio并发数_asyncio aiohttp限制并发数量和超时时间

importtimeimportqueueimportrandomimportasyncioimporttracebackimportcollectionsfrom aiohttp importClientSession, TCPConnector, client_exceptions, ClientTimeout

queue_data=queue.Queue()

timeout_domains=[]

unknown_error_domains=[]

start_time_list=[]

asyncdeffetch(session, n, url):""":param session: aiohttp.ClientSession

:param n: task编号

:param url: 请求url"""start_time=time.time()#noinspection PyBroadException

try:

async with session.get(url=url, verify_ssl=False) as response:

r=await response.read()

end_time=time.time()

cost= end_time -start_time

msg= "第{}个查询请求，花费时间: {}s, 返回信息: {}\n".format(n, cost, r.decode('unicode-escape'))#print(msg)

queue_data.put(1)exceptclient_exceptions.ServerTimeoutError as timeout_error:print("request timeout error: {}, url: {}".format(timeout_error, url))

timeout_domains.append(url)exceptException:print("request unknown error: {}".format(traceback.format_exc()))

unknown_error_domains.append(url)

start_time_list.append(str(start_time).split(".")[0])

asyncdefchunks(sem, session, i, url):"""限制并发数"""async with sem:

await fetch(session, i, url)defget_domains():

urls=[]for _ in range(1000):

urls.append("http://127.0.0.1:5000/?delay={}".format(random.randint(1, 8)))returnurls

asyncdefmain(urls):

sem= asyncio.Semaphore(400)

timeout= ClientTimeout(total=10, connect=2, sock_connect=15, sock_read=5)

async with ClientSession(connector=TCPConnector(limit=400), timeout=timeout) as session:

tasks= [asyncio.create_task(chunks(sem, session, index, url)) for index, url inenumerate(urls)]

await asyncio.wait(tasks)if __name__ == '__main__':

domains=get_domains()

asyncio.run(main(domains))print("success number: {}, timeout number: {}, unknown_error number: {}".format(queue_data.qsize(),

len(timeout_domains),

len(unknown_error_domains)))print(sorted(collections.Counter(start_time_list).items(), key=lambdaitem:item[0]))#1. 没有超时的，第一批400个同一秒发起， 再往后就看response相应与读取速度

#success number: 1000, timeout number: 0, unknown_error number: 0

#[('1593246892', 400), ('1593246894', 48), ('1593246895', 55), ('1593246896', 55), ('1593246897', 76),

#('1593246898', 74), ('1593246899', 90), ('1593246900', 96), ('1593246901', 106)]

#2. 有超时的

#success number: 517, timeout number: 483, unknown_error number: 0

#[('1593248067', 400), ('1593248068', 36), ('1593248069', 43), ('1593248070', 75), ('1593248071', 64),

#('1593248072', 168), ('1593248073', 126), ('1593248074', 69), ('1593248075', 19)]