python 简单并发下载器

python 简单并发下载器

阻塞式下载器

import gevent
import urllib.request

def my_downLoad(url):
	print('GET : %s'%url)
	resp=urllib.request.urlopen(url)
	data=resp.read()
	print("获取到 %d byte 从 %s"%(len(data),url))

gevent.joinall([
	gevent.spawn(my_downLoad,'http://www.baidu.com'),
	gevent.spawn(my_downLoad,'https://www.jd.com/?cu=true&utm_source=hao.360.com&utm_medium=tuiguang&utm_campaign=t_1000003625_360mz&utm_term=7da5bfb212bf4c939d91daafc210bcd0'),
	gevent.spawn(my_downLoad,'http://www.360kuai.com/pc/detail?url=http%3A%2F%2Fzm.news.so.com%2F60d7851de9479e607fc07952f03f4541&check=f02d72bdf5834394&ctype=gallery&sign=360_37ccb7e5&scene=21001&sub_scene=1'),
	])

协程式并发下载器

from gevent import monkey
import gevent
import urllib.request
#猴子补丁
monkey.patch_all()# 将程序中⽤到的耗时操作的代码，换为gevent中⾃⼰实现的模块

def my_downLoad(url):
	print('GET : %s'%url)
	resp=urllib.request.urlopen(url)
	data=resp.read()
	print("获取到 %d byte 从 %s"%(len(data),url))

gevent.joinall([
	gevent.spawn(my_downLoad,'http://www.baidu.com'),
	gevent.spawn(my_downLoad,'https://www.jd.com/?cu=true&utm_source=hao.360.com&utm_medium=tuiguang&utm_campaign=t_1000003625_360mz&utm_term=7da5bfb212bf4c939d91daafc210bcd0'),
	gevent.spawn(my_downLoad,'http://www.360kuai.com/pc/detail?url=http%3A%2F%2Fzm.news.so.com%2F60d7851de9479e607fc07952f03f4541&check=f02d72bdf5834394&ctype=gallery&sign=360_37ccb7e5&scene=21001&sub_scene=1'),

	])

通过猴子补丁的形式将阻塞式变为协程式，提高了运行效率节约了运行时间
猴子补丁了解传送门：https://blog.csdn.net/qq_38839677/article/details/82288598

实现多个网页下载

from gevent import monkey
import gevent
import urllib.request
#猴子补丁
monkey.patch_all()

def my_downLoad(file_name,url):
	print('GET : %s'%url)
	resp=urllib.request.urlopen(url)
	data=resp.read()
	print("获取到 %d byte 从 %s"%(len(data),url))
	f= open(file_name,'wb')
	f.write(data)
	f.close()

gevent.joinall([
	gevent.spawn(my_downLoad,'1.html','http://www.baidu.com'),
	gevent.spawn(my_downLoad,'2.html','https://www.jd.com/?cu=true&utm_source=hao.360.com&utm_medium=tuiguang&utm_campaign=t_1000003625_360mz&utm_term=7da5bfb212bf4c939d91daafc210bcd0'),
	gevent.spawn(my_downLoad,'3.html','http://www.360kuai.com/pc/detail?url=http%3A%2F%2Fzm.news.so.com%2F60d7851de9479e607fc07952f03f4541&check=f02d72bdf5834394&ctype=gallery&sign=360_37ccb7e5&scene=21001&sub_scene=1'),

	])