多进程:
#!/usr/bin/env python
#coding:gbk
#================================================================
#Name : GetFileFromWeb.py
#Desc : Get File From Web
#Author : zalzhou
#Date : 20121031
#Modify :
#From :
#Desc : 同步10个进程,md5列表分割10份
#================================================================
import os
import sys
import time
import urllib
import random
import multiprocessing
md5List = []
Num = 10
def GetMd5List():
for i in range(Num):
tmpList = []
md5List.append(tmpList)
curPath = os.path.abspath(os.path.curdir)
filePath = curPath + "\\md5.txt"
f = open(filePath,'r')
for line in f.readlines():
i = random.randint(0,9)
md5List[i].append(line.strip())
f.close
def GetFileFromList(md5List):
for md5 in md5List:
GetFileFromMd5(md5)
def GetFileFromMd5(md5):
curPath = os.path.abspath(os.path.curdir)
for i in range(5):
j = random.randint(0,2)
if j == 0:
try:
local = os.path.join(curPath,md5)
url = r'http://123.138.163.150/download?md5='+md5+'&start=0&end=0&key=NONE'
urllib.urlretrieve(url,local,)
print "get file\r\n"
break
except:
print "try again\r\n"
continue
if j == 1:
try:
local = os.path.join(curPath,md5)
url = r'http://123.138.163.151/download?md5='+md5+'&start=0&end=0&key=NONE'
urllib.urlretrieve(url,local,)
print "get file\r\n"
break
except:
print "try again\r\n"
continue
if j == 2:
try:
local = os.path.join(curPath,md5)
url = r'http://123.138.163.152/download?md5='+md5+'&start=0&end=0&key=NONE'
urllib.urlretrieve(url,local,)
print "get file\r\n"
break
except:
print "try again\r\n"
continue
time.sleep(1)
if __name__ == '__main__':
GetMd5List()
for i in range(Num):
p = multiprocessing.Process(target=GetFileFromList,args=(md5List[i],))
p.start()
#p.join()
进程池:
#!/usr/bin/env python
#coding:gbk
#================================================================
#Name : GetFileFromWeb.py
#Desc : Get File From Web
#Author : zalzhou
#Date : 20121031
#Modify :
#From :
#Desc : 异步10个进程,md5列表分割10份
#================================================================
import os
import sys
import time
import urllib
import random
import multiprocessing
md5List = []
Num = 10
def GetMd5List():
for i in range(Num):
tmpList = []
md5List.append(tmpList)
curPath = os.path.abspath(os.path.curdir)
filePath = curPath + "\\md5.txt"
f = open(filePath,'r')
for line in f.readlines():
i = random.randint(0,9)
md5List[i].append(line.strip())
f.close
def GetFileFromList(md5List):
for md5 in md5List:
GetFileFromMd5(md5)
def GetFileFromMd5(md5):
curPath = os.path.abspath(os.path.curdir)
for i in range(5):
j = random.randint(0,2)
if j == 0:
try:
local = os.path.join(curPath,md5)
url = r'http://123.138.163.150/download?md5='+md5+'&start=0&end=0&key=NONE'
urllib.urlretrieve(url,local,)
print "get file\r\n"
break
except:
print "try again\r\n"
continue
if j == 1:
try:
local = os.path.join(curPath,md5)
url = r'http://123.138.163.151/download?md5='+md5+'&start=0&end=0&key=NONE'
urllib.urlretrieve(url,local,)
print "get file\r\n"
break
except:
print "try again\r\n"
continue
if j == 2:
try:
local = os.path.join(curPath,md5)
url = r'http://123.138.163.152/download?md5='+md5+'&start=0&end=0&key=NONE'
urllib.urlretrieve(url,local,)
print "get file\r\n"
break
except:
print "try again\r\n"
continue
time.sleep(1)
if __name__ == '__main__':
GetMd5List()
proc_pool = multiprocessing.Pool(Num)
pmap = dict(proc_pool.map(GetFileFromList,md5List,len(md5List)/Num))
proc_pool.close()
#proc_pool.join()
多线程:
#!/usr/bin/env python
#coding:gbk
#================================================================
#Name : GetFileFromWeb.py
#Desc : Get File From Web
#Author : zalzhou
#Date : 20121031
#Modify :
#From :
#Desc : 同步10个线程,md5列表分割10份
#================================================================
import os
import sys
import time
import string
import urllib
import random
import threading
md5List = []
Num = 10
def GetMd5List():
for i in range(Num):
tmpList = []
md5List.append(tmpList)
curPath = os.path.abspath(os.path.curdir)
filePath = curPath + "\\md5.txt"
f = open(filePath,'r')
for line in f.readlines():
j = random.randint(0,9)
md5List[j].append(line.strip())
f.close
class GetFileFromWeb(threading.Thread):
def __init__(self,md5List):
threading.Thread.__init__(self)
self.md5List = md5List
self.curPath = os.path.abspath(os.path.curdir)
def GetFileFromMd5(self,md5):
local = os.path.join(self.curPath,md5)
for i in range(5):
j = random.randint(0,2)
if j == 0:
try:
url = r'http://123.138.163.150/download?md5='+md5+'&start=0&end=0&key=NONE'
urllib.urlretrieve(url,local,)
print "get file\r\n"
break
except:
print "try again\r\n"
continue
elif j == 1:
try:
url = r'http://123.138.163.151/download?md5='+md5+'&start=0&end=0&key=NONE'
urllib.urlretrieve(url,local,)
print "get file\r\n"
break
except:
print "try again\r\n"
continue
elif j == 2:
try:
url = r'http://123.138.163.152/download?md5='+md5+'&start=0&end=0&key=NONE'
urllib.urlretrieve(url,local,)
print "get file\r\n"
break
except:
print "try again\r\n"
continue
def run(self):
for md5 in self.md5List:
self.GetFileFromMd5(md5)
if __name__ == '__main__':
GetMd5List()
for i in range(Num):
t = GetFileFromWeb(md5List[i])
t.start()
线程池:
#!/usr/bin/env python
#coding:gbk
#================================================================
#Name : GetFileFromWeb.py
#Desc : Get File From Web
#Author : zalzhou
#Date : 20121031
#Modify :
#From :
#Desc : 异步10个线程,md5列表分割20份
#================================================================
import os
import sys
import time
import urllib
import random
import threading
import Queue
md5List = []
Num = 10
queue = Queue.Queue()
def GetMd5List():
for i in range(20):
tmpList = []
md5List.append(tmpList)
curPath = os.path.abspath(os.path.curdir)
filePath = curPath + "\\md5.txt"
f = open(filePath,'r')
for line in f.readlines():
i = random.randint(0,19)
md5List[i].append(line.strip())
f.close
class GetFileFromWeb(threading.Thread):
def __init__(self,queue):
threading.Thread.__init__(self)
self.queue = queue
def run(self):
while True:
md5List = self.queue.get()
self.Work(md5List)
self.queue.task_done()
def GetFileFromMd5(self,md5):
curPath = os.path.abspath(os.path.curdir)
for i in range(5):
j = random.randint(0,2)
if j == 0:
try:
local = os.path.join(curPath,md5)
url = r'http://123.138.163.150/download?md5='+md5+'&start=0&end=0&key=NONE'
urllib.urlretrieve(url,local,)
print "get file\r\n"
break
except:
print "try again\r\n"
continue
if j == 1:
try:
local = os.path.join(curPath,md5)
url = r'http://123.138.163.151/download?md5='+md5+'&start=0&end=0&key=NONE'
urllib.urlretrieve(url,local,)
print "get file\r\n"
break
except:
print "try again\r\n"
continue
if j == 2:
try:
local = os.path.join(curPath,md5)
url = r'http://123.138.163.152/download?md5='+md5+'&start=0&end=0&key=NONE'
urllib.urlretrieve(url,local,)
print "get file\r\n"
break
except:
print "try again\r\n"
continue
time.sleep(1)
def Work(self,md5List):
for md5 in md5List:
self.GetFileFromMd5(md5)
if __name__ == '__main__':
GetMd5List()
for i in range(Num):
t = GetFileFromWeb(queue)
t.setDaemon(True)
t.start()
for md5 in md5List:
queue.put(md5)
queue.join()