一、单线程
import requests
from Queue import Queue
from lxml import etree
import time
class Douban(object):
def __init__(self):
self.start_urls = ["https://movie.douban.com/top250?start=" + str(num) for num in range(0, 226, 25)]
self.headers = {"User-Agent" : "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko"}
self.data_queue = Queue()
def send_request(self, url):
print "[INFO]: 正在处理%s " % url
response = requests.get(url, headers = self.headers)
time.sleep(1)
self.parse_page(response)
def parse_page(self, response):
html_obj = etree.HTML(response.content)
node_list = html_obj.xpath("//div[@class='info']")
for node in node_list:
title = node.xpath("./div[@class='hd']/a/span[1]/text()")[0]
score = node.xpath("./div[@class='bd']//span[@class='rating_num']/text()")[0]
self.data_queue.put(score + "\t" + title)
def start_work(self):
for url in self.start_urls:
self.send_request(url)
while not self.data_queue.empty():
print self.data_queue.get()
if __name__ == "__main__":
start = time.time()
spider = Douban()
spider.start_work()
print "[INFO]: Useing time %f" % (time.time() - start)
二、多线程 threading
import requests
from Queue import Queue
from lxml import etree
import time
import threading
class Douban(object):
def __init__(self):
self.start_urls = ["https://movie.douban.com/top250?start=" + str(num) for num in range(0, 226, 25)]
self.headers = {"User-Agent" : "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko"}
self.data_queue = Queue()
def send_request(self, url):
print "[INFO]: 正在处理%s " % url
response = requests.get(url, headers = self.headers)
time.sleep(1)
self.parse_page(response)
def parse_page(self, response):
html_obj = etree.HTML(response.content)
node_list = html_obj.xpath("//div[@class='info']")
for node in node_list:
title = node.xpath("./div[@class='hd']/a/span[1]/text()")[0]
score = node.xpath("./div[@class='bd']//span[@class='rating_num']/text()")[0]
self.data_queue.put(score + "\t" + title)
def start_work(self):
"""
for url in self.start_urls:
self.send_request(url)
"""
thread_list = []
for url in self.start_urls:
thread = threading.Thread(target = self.send_request, args = [url])
thread.start()
thread_list.append(thread)
for thread in thread_list:
thread.join()
while not self.data_queue.empty():
print self.data_queue.get()
if __name__ == "__main__":
start = time.time()
spider = Douban()
spider.start_work()
print "[INFO]: Useing time %f" % (time.time() - start)
三、多线程:multiprocessing.dummy
import requests
from Queue import Queue
from lxml import etree
import time
from multiprocessing.dummy import Pool
class Douban(object):
def __init__(self):
self.start_urls = ["https://movie.douban.com/top250?start=" + str(num) for num in range(0, 226, 25)]
self.headers = {"User-Agent" : "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko"}
self.data_queue = Queue()
def send_request(self, url):
print "[INFO]: 正在处理%s " % url
response = requests.get(url, headers = self.headers)
time.sleep(1)
self.parse_page(response)
def parse_page(self, response):
html_obj = etree.HTML(response.content)
node_list = html_obj.xpath("//div[@class='info']")
for node in node_list:
title = node.xpath("./div[@class='hd']/a/span[1]/text()")[0]
score = node.xpath("./div[@class='bd']//span[@class='rating_num']/text()")[0]
self.data_queue.put(score + "\t" + title)
def start_work(self):
"""
for url in self.start_urls:
self.send_request(url)
"""
"""
thread_list = []
for url in self.start_urls:
thread = threading.Thread(target = self.send_request, args = [url])
thread.start() # run
thread_list.append(thread)
# 让主线程等待所有的子线程结束,主线程再继续执行
for thread in thread_list:
thread.join()
"""
pool = Pool(len(self.start_urls))
pool.map(self.send_request, self.start_urls)
pool.close()
pool.join()
while not self.data_queue.empty():
print self.data_queue.get()
if __name__ == "__main__":
start = time.time()
spider = Douban()
spider.start_work()
print "[INFO]: Useing time %f" % (time.time() - start)
四、协程 gevent
import requests
from Queue import Queue
from lxml import etree
import time
import gevent
from gevent import monkey
monkey.patch_all()
class Douban(object):
def __init__(self):
self.start_urls = ["https://movie.douban.com/top250?start=" + str(num) for num in range(0, 226, 25)]
self.headers = {"User-Agent" : "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko"}
self.data_queue = Queue()
def send_request(self, url):
print "[INFO]: 正在处理%s " % url
response = requests.get(url, headers = self.headers)
time.sleep(1)
self.parse_page(response)
def parse_page(self, response):
html_obj = etree.HTML(response.content)
node_list = html_obj.xpath("//div[@class='info']")
for node in node_list:
title = node.xpath("./div[@class='hd']/a/span[1]/text()")[0]
score = node.xpath("./div[@class='bd']//span[@class='rating_num']/text()")[0]
self.data_queue.put(score + "\t" + title)
def start_work(self):
"""
for url in self.start_urls:
self.send_request(url)
"""
"""
thread_list = []
for url in self.start_urls:
thread = threading.Thread(target = self.send_request, args = [url])
thread.start() # run
thread_list.append(thread)
# 让主线程等待所有的子线程结束,主线程再继续执行
for thread in thread_list:
thread.join()
"""
"""
# 创建线程池对象
pool = Pool(len(self.start_urls))
# 依次执行start_urls里的每个url地址请求
pool.map(self.send_request, self.start_urls)
# 关闭线程池
pool.close()
# 让主线程等待所有子线程结束,主线程再继续执行
pool.join()
"""
"""
job_list = []
for url in self.start_urls:
# 创建一个协程任务
job = gevent.spawn(self.send_request, url)
# 将协程任务放入任务队列里
job_list.append(job)
# 获取所有的协程任务,并放入任务队列
gevent.joinall(job_list)
"""
job_list = [gevent.spawn(self.send_request, url) for url in self.start_urls]
gevent.joinall(job_list)
while not self.data_queue.empty():
print self.data_queue.get()
if __name__ == "__main__":
start = time.time()
spider = Douban()
spider.start_work()
print "[INFO]: Useing time %f" % (time.time() - start)