批量查询URL是否被收录(Python多线程)

#coding=utf-8
import urllib,urllib2,re,random


def get_proxies():
    html = urllib2.urlopen('http://www.xici.net.co/nn/').read()
    pa = re.compile(r'<td>(\d+.\d+.\d+.\d+)</td>[\s\S]+?<td>(\d+)</td>[\s\S]+?<div title="(\d\.\d+?)秒" class="bar">')
    ip_list = pa.findall(html)
    return [x for x in ip_list if x[2] < '3']


ip_list = get_proxies()
print 'collect %s ip' % len(ip_list)


def is_index(line): 
    while 1:
        try:
            ip = random.choice(ip_list)
            proxies = {}
            proxies['http'] = 'http://%s:%s' % (ip[0], ip[1])
            proxy_handler = urllib2.ProxyHandler(proxies)
            opener = urllib2.build_opener(proxy_handler)
            html=opener.open('http://www.baidu.com/s?wd=%s' % urllib.quote_plus(line), timeout=15).read()
            if '="http://verify.baidu.com' in html:
                ip_list.remove(ip)
                print ("yzm show", ip[0])
                continue
            if '<div class="nors"><p>抱歉，没有找到与' in html or 'class=f14>没有找到该URL。您可以直接访问' in html:
                return (0, ip[0])
            else:
                return (1, ip[0])
        except:
            ip_list.remove(ip)
            is_index(line)
            
for line in open('url.txt'):
    line = line.strip()
    print line + '\t' + '%s\t%s' % is_index(line)