grequests使用方式
实例化请求对象: grequests.request(method, url, **kwargs)
发起请求获得响应: grequests.map(requests, stream=False, size=None, exception_handler=None, gtimeout=None)
将请求的生成器对象并发地转换为反应的产生者:grequests.imap(requests, stream=False, size=2, exception_handler=None)
import json
import ast
def test_1():
# grequests的使用方式
adata = json.dumps({"key": "value"})
header = {"Content-type": "appliaction/json", "Accept": "application/json"}
url = "http://www.baidu.com"
task = []
req = grequests.request("POST", url=url, data=adata, headers=header)
task.append(req)
resp = grequests.map(task, size=3)
for j in resp:
print(ast.literal_eval(json.dumps(j))[0]["status_code"])
def test_2():
# grequests方式
start_time = time.time()
url = "http://www.baidu.com"
tasks = []
for _ in range(10000):
tasks.append(grequests.get(url))
# size为50
if len(tasks) == 50:
resp = grequests.imap(tasks)
for j in resp:
print(j.status_code)
del tasks[0:]
end_time = time.time()
print(f"耗时:{end_time - start_time}.....") # 203.33479380607605
import requests
def test_3():
# requests直接请求
start_time = time.time()
url = "http://www.baidu.com"
for _ in range(10000):
resp = requests.get(url)
print(resp.status_code)
end_time = time.time()
print(f"耗时:{end_time - start_time}.....") # 258.4600441455841
def test_4():
# request+gevent的方式
import gevent
import time
import requests
from gevent import monkey
monkey.patch_all()
task = []
def func():
url = "http://www.baidu.com"
resp = requests.get(url=url)
print(resp.status_code)
start_time = time.time()
for _ in range(1, 10001):
task.append(gevent.spawn(func))
resp = gevent.joinall(task)
print(len(resp))
end_time = time.time()
print(f"耗时:{end_time - start_time}.....") # 38.15373682975769