多线程 + 队列

import re
import time
import requests
from threading import Thread
from queue import Queue


class Spider(Thread):
    def __init__(self, url_queue):
        Thread.__init__(self)
        self.url_queue = url_queue

    def run(self):
        while not self.url_queue.empty():
            url = self.url_queue.get()
            response = requests.get(url).content.decode('utf-8')
            title = re.findall(r'<title>(.*?)</title>', response)[0]
            print(title)



if __name__ == '__main__':
    t11 = time.time()
    url_list = ['http://www.baidu.com' for _ in range(50)]

    url_queue = Queue()

    for url in url_list:
        url_queue.put(url)

    t_list = []
    for _ in range(10):
        t1 = Spider(url_queue)
        t_list.append(t1)
        t1.start()

    # join 阻塞主进程
    for t in t_list:
        t.join()

    print(time.time()-t11)