python并发发送http请求_python3-使用进程方式进行并发请求

1：使用HTTP长连接的方式进行发送请求:

代码如下:

import os

import requests

from requests.adapters import HTTPAdapter

from requests.exceptions import ConnectionError

class AliveRequests(object):

# 会话池

session_pool = {}

def __enter__(self):

self.session()

return self

def __exit__(self, *excinfo):

pass

@staticmethod

def session():

cur_pid = os.getpid()

if cur_pid not in list(AliveRequests.session_pool.keys()):

session = requests.session()

session.mount('http://', HTTPAdapter(pool_maxsize=1000, pool_block=True))

AliveRequests.session_pool[cur_pid] = session

return AliveRequests.session_pool[cur_pid]

@staticmethod

def request(method=None, url=None, headers=None, files=None, data=None, params=None, auth=None, cookies=None, hooks=None, json=None, timeout=None):

session = AliveRequests.session()

request = requests.Request(method, url, headers, files, data, params, auth, cookies, hooks, json)

while True:

try:

with session.send(session.prepare_request(request), timeout=(10, timeout)) as response:

return response

except ConnectionError as err:

err_obj = err.message if isinstance(err.message, Exception) else err

if "BadStatusLine" in err_obj.message or "reset by peer" in err_obj.message:

continue

else:

raise err

except Exception as ex:

raise ex

并发请求:

from multiprocessing import Pool

def work(self):

datas = []

po = Pool(n)  # 设置进程数

res_list = po.map(func(函数名), 参数列表)   # 传入参数列表，传递给函数时参数列表遍历之后将每个元素依此传递，

for res in iter(res_list):

accessToken = res[0].json()["data"]["access_token"] if res[0].status_code == 200 and "data" in res[0].json() else ""

dict_s = {"name": res[2][0],

"accessToken":accessToken,

"user_id":res[2][1],

"time": res[1]}

datas.append(dict_s)

return datas

统计发起请求的事件，单位(s)

resp = AliveRequests.request('POST', self.login_url, params=params, headers=self.headers)

resp.elapsed.total_seconds()