python 多线程处理抓取网页

不使用多线程，直接抓取接口

import urllib2
import time
from BeautifulSoup import BeautifulSoup

hosts = ["http://yahoo.com", "http://google.com", "http://amazon.com",
"http://ibm.com", "http://www.microsoft.com/en-us/default.aspx"]

start = time.time()
#grabs urls of hosts and prints first 1024 bytes of page
for host in hosts:
    url = urllib2.urlopen(host)
    chunk = url.read()

    soup = BeautifulSoup(chunk)
    print soup.findAll(['title'])

print "Elapsed Time: %s" % (time.time() - start)

使用多线程优化

#!/usr/bin/env python
#!/usr/bin/env python
import Queue
import threading
import urllib2
import time
from BeautifulSoup import BeautifulSoup

hosts = ["http://yahoo.com", "http://google.com", "http://amazon.com",
"http://ibm.com", "http://www.microsoft.com/en-us/default.aspx"]

queue = Queue.Queue()

class ThreadUrl(threading.Thread):
	"""Threaded Url Grab"""
	def __init__(self, queue):
		threading.Thread.__init__(self)
		self.queue = queue

	def run(self):
		while True:
			#grabs host from queue
			host = self.queue.get()

			#grabs urls of hosts and prints first 1024 bytes of page
			url = urllib2.urlopen(host)
			chunk = url.read()            
			soup = BeautifulSoup(chunk)
			print soup.findAll(['title'])

			#signals to queue job is done
			self.queue.task_done()



def main():
	#spawn a pool of threads, and pass them queue instance 
	for i in range(5):
		t = ThreadUrl(queue)
		t.setDaemon(True)
		t.start()

	#populate queue with data   
	for host in hosts:
		queue.put(host)

	#wait on the queue until everything has been processed     
	queue.join()

start = time.time()
main()
print "Elapsed Time: %s" % (time.time() - start)

引入多个队列，优化获取和处理两个步骤

import Queue
import threading
import urllib2
import time
from BeautifulSoup import BeautifulSoup

hosts = ["http://yahoo.com", "http://google.com", "http://amazon.com",
        "http://ibm.com","http://www.microsoft.com/en-us/default.aspx"]

queue = Queue.Queue()
out_queue = Queue.Queue()

class ThreadUrl(threading.Thread):
    """Threaded Url Grab"""
    def __init__(self, queue, out_queue):
        threading.Thread.__init__(self)
        self.queue = queue
        self.out_queue = out_queue

    def run(self):
        while True:
            #grabs host from queue
            host = self.queue.get()

            #grabs urls of hosts and then grabs chunk of webpage
            url = urllib2.urlopen(host)
            chunk = url.read()

            #place chunk into out queue
            self.out_queue.put(chunk)

            #signals to queue job is done
            self.queue.task_done()

class DatamineThread(threading.Thread):
    """Threaded Url Grab"""
    def __init__(self, out_queue):
        threading.Thread.__init__(self)
        self.out_queue = out_queue

    def run(self):
        while True:
            #grabs host from queue
            try:
                chunk = self.out_queue.get()
            except TypeError as e:
                print e
            else:
                #parse the chunk
                soup = BeautifulSoup(chunk)
                print soup.findAll(['title'])

            #signals to queue job is done
            self.out_queue.task_done()

def main():

    #spawn a pool of threads, and pass them queue instance
    for i in range(5):
        t = ThreadUrl(queue, out_queue)
        t.setDaemon(True)
        t.start()

    #populate queue with data
    for host in hosts:
        queue.put(host)

    for i in range(5):
        dt = DatamineThread(out_queue)
        dt.setDaemon(True)
        dt.start()


    #wait on the queue until everything has been processed
    queue.join()
    out_queue.join()

start = time.time()
main()
print "Elapsed Time: %s" % (time.time() - start)

http://www.ibm.com/developerworks/cn/aix/library/au-threadingpython/

http://yiminghe.iteye.com/blog/673524

http://isilic.iteye.com/category/256210

http://www.cnblogs.com/holbrook/archive/2012/03/21/2410120.html