python 多线程采集网页完善版

import threading,time,random,htmllib,urllib,formatter,string,re

def getPageNum(data):
    result=re.findall( r'pageNum">共(\d+)' , data )
    try:
        return result[0]
    except:
        return 0

def geturls(cururl):
    global l
    data = urllib.urlopen(cururl).read()
    linkdemo = GetLinks()
    linkdemo.feed(data)
    linkdemo.close()
    urls=[]
    for item in linkdemo.links:
        try:            
            if item.index('/suppliers/') and item.startswith(cururl)==False:
                urls.append(item)
        except ValueError:
            pass

    cursize=getPageNum(data)
    if int(cursize)>100:
        cursize=100
    output = open('datadown/listUrl.txt', 'a+')
    output.write(cururl + "@@@@" + str(cursize) + "\n")
    output.close()        

    for ni in urls:        
        try:
            l.index(ni)
            pass                
        except ValueError:                
            l.append(ni)
            output = open('datadown/historyUrl.txt', 'a+')
            output.write(ni + "\n")
            output.close()


class GetLinks(htmllib.HTMLParser):
    def __init__(self):
        self.links = []
        f = formatter.NullFormatter()
        htmllib.HTMLParser.__init__(self, f)  
 
    def anchor_bgn(self, href, name, type):
        self.save_bgn()  
        self.link = href  
 
    def anchor_end(self):
        text = string.strip(self.save_end())
        if self.link and text:  
            self.links.append(self.link)    


        
mylock = threading.RLock()
num=0
l=[]
   
class myThread(threading.Thread):  
    def __init__(self, name):  
        threading.Thread.__init__(self)  
        self.t_name = name
          
    def run(self):  
        global num,l        
        while True:
            #同步开始
            mylock.acquire()
            num+=1
            if len(l)==0:
                curl="http://product.cn.china.cn/suppliers/%B9%E3%D6%DD/"
                print '当前线程：%s，当前连接：%s，当前数量：%d\n'%(self.t_name, curl, num)
                geturls(curl)

            curl=l[num-1]
            print '当前线程：%s，当前连接：%s，当前数量：%d\n'%(self.t_name, curl, num+1)
            mylock.release()
            #同步结束
            geturls(curl)
            
            if num>=len(l):  
                #mylock.release()
                #print 'len(l):'+ str(len(self.web.l))
                print "\n\nl:\n"
                #for i in l:
                    #print i                
                break             
 
            
def test():
    for i in range(1,6):
        threadi=myThread('A'+str(i))
        threadi.start()

if __name__== '__main__':
    test()