亲测通过
import urllib2,os from HTMLParser import HTMLParser import urllib import oss2 import MySQLdb import socket from multiprocessing import Pool import traceback from multiprocessing.dummy import Pool as ThreadPool def save_pic(url,filename): count=0 while True: try: print filename,'begin...' urllib.urlretrieve(url, filename) except socket.timeout: print filename,'count:',count count+=1 except Exception,e: print filename,'other',e count+=1 else: print filename,'successfully!' break if __name__ == "__main__": access_key_id = os.getenv('OSS_TEST_ACCESS_KEY_ID', '**********') access_key_secret = os.getenv('OSS_TEST_ACCESS_KEY_SECRET', '**************') bucket_name = os.getenv('OSS_TEST_BUCKET', '**********') endpoint = os.getenv('OSS_TEST_ENDPOINT', '*************') host = 'localhost' user = 'root' pwd = 'root' db = 'test' conn = MySQLdb.connect(host,user,pwd,db,charset='utf8') cursor = conn.cursor() sql='select * from ks_table' img_dir = "D:\\images" if not os.path.isdir(img_dir): os.mkdir(img_dir) os.chdir(img_dir) print os.getcwd() url = "" try: # execute sql cursor.execute(sql) # fetch records results = cursor.fetchall() print 'Parent process %s.' % os.getpid() pool=ThreadPool(50) for row in results: xuexinid = row[0] questionurl=row[1] url = "http://"+questionurl.replace("@", ".*******.aliyuncs.com/"); filename = os.path.basename(url) filename=xuexinid+'_'+filename pool.apply_async(save_pic,args=(url,filename)) pool.close() pool.join() print 'all successfully!' except Exception,e: print "Error: unable to fecth data",e.message print traceback.print_exc() cursor.close() conn.close()