python下载批量图片

参考http://www.jianshu.com/p/1f63fa9d1c20

需要下载数据集，尝试自己写，使用multiprocessing库中的Pool类，对代码小作修改：

import urllib
import time
import socket
from multiprocessing import Pool
def save_pic(pic_url,pic_name):
count=0
while True:
try:
print 'picture',pic_name,'begin...'
socket.setdefaulttimeout(120)
urllib.urlretrieve(pic_url,pic_name)
except socket.timeout:
print 'timeout:',pig_name,'count:',count
count+=1
if count==5:
break
except Exception,e:
print pig_name,'other fault:',e
count+=1
else:
print 'picture',pic_name,'save successfully!it has tried to download',count,'times'
break;

def save_pics(input,outputDir):
f=open(input,"r")
count=0
total=10
urls=[]
while 1:
line=f.readline()
count+=1
if line:
url=line.split(",")
temp=url[0]
url[0]=outputDir+temp+".jpg"
urls.append(url)
else:
break
f.close
'''for item in urls:
print(item)'''
p=Pool(40)
count=0
totalNum=len(urls)
for item in urls:
print item[0],item[1]
count+=1
print 'totalNum is ',totalNum,' now ',count,'/',totalNum,'=',100.0 * count/totalNum
p.apply_async(save_pic,args=(item[1],item[0]))
p.close()
p.join()
print 'all save successfully'

if __name__=='__main__':
input="D:\\url_path\\photos.txt"
outputDir="D:\\img_path\\"
save_pics(input,outputDir)

下载效率提升很多，42万张的数据集，一天下完。