import grequests
batch_reqs =[]
batch_result =[]for push_data in push_datas:
datas ={'key': push_data['url'],"bucket":0}
batch_reqs.append(
grequests.post(url=url,
json=datas,
headers=headers))
res_list = grequests.map(batch_reqs)# 并行发送,等最后一个运行完后返回for i, res inenumerate(res_list):
data =""
ret =-1try:if res and res.status_code ==200:
ret =0
res.encoding ='utf8'
result = json.loads(res.text)if result.get('object')isnotNone:
data = unquote(result['object'])except(KeyError, ValueError)as e:
ret =-1
log.error("docid={}|cos_err|data={}".format(push_datas[i]["docid"], push_datas[i]))
batch_result.append((ret, data))