python多线程下载小说章节重复_python多线程处理重复任务10000次

#!/usr/bin/python

import threading

import urllib2

#f = open("/root/wget.txt","a")

x = [ i.strip() for i in open("/root/ceshi/abc.txt","r") ]

#f1 = open("/root/abc.txt","r")

def wget(x):

response = urllib2.urlopen("http://abc.abc.com/organize/domain/allupdate?domainname=%s"%x)

if response.read() == '{"code": "0", "data": "ok"}':

f.write('%s {"code": "0", "data": "ok"}\n'%x)

else:

f.write('%s {"code": "0", "data": "fail"}\n'%x)

def main():

n = 0

while len(x) - n >= 20:

f = open("/root/ceshi/wget.txt","a")

threads = []

for i in range(n+0,n+20):

t = threading.Thread(target=wget,args=(x[i],))

threads.append(t)

for i in range(20):

threads[i].start()

for i in range(20):

threads[i].join()

n+=20

f.close()

f = open("/root/ceshi/wget.txt","a")

for i in range(n,len(x)):

threads = []

t = threading.Thread(target=wget,args=(x[i]))

threads.append(t)

for i in range(len(x)-n):

threads[i].start()

for i in range(len(x)-n):

threads[i].join()

f.close()

if __name__ == '__main__':

main()