协程在爬虫上的应用
from gevent import monkey
monkey.patch_all()
import gevent
import requests
import time
def get_page(url):
print(‘GET: %s’ % url)
time.sleep(2)
response = requests.get(url)
if response.status_code == 200:
print(’%d bytes received from %s’ % (len(response.text), url))
start_time = time.time()
gevent.joinall([
gevent.spawn(get_page, ‘https://www.cnblogs.com/russellyoung/p/python-zhi-xie-cheng.html’),
gevent.spawn(get_page, ‘https://www.adidas.com.cn/item-subscribe’),
# gevent.spawn(get_page, ‘https://github.com/’),
])
stop_time = time.time()
print(‘run time is %s’ % (stop_time - start_time))
print(’----------------------------------------------------------------------------------------’)
s = time.time()
get_page(‘https://www.cnblogs.com/russellyoung/p/python-zhi-xie-cheng.html’)
get_page(‘https://www.adidas.com.cn/item-subscribe’)
requests.get(‘https://github.com/’)
t = time.time()
print(‘串行时间>>’,t-s) # run time is 2.5960400104522705