DataOutPut.py
#!/usr/bin/python # -*- coding: UTF-8 -*- import codecs import time class DataOutPut(object): def __init__(self): self.filepath = 'baike_%s.html' % (time.strftime("%Y_%m_%d_%H_%M_%S", time.localtime())) self.output_head(self.filepath) self.datas = [] def store_data(self, data): if data is None: return self.datas.append(data) if len(self.datas) > 10: self.output_html(self.filepath) def output_head(self, path): ''' 将HTML头写进去 :return: ''' fout = codecs.open(path, 'w', encoding='utf-8') fout.write("<html>") fout.write(r'''<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />''') fout.write("<body>") fout.write("<table>") fout.close() def output_html(self, path): ''' 将数据写入HTML文件中 :param path: 文件路径 :return: ''' fout = codecs.open(path, 'a', encoding='utf-8') for data in self.datas: fout.write("<tr>") fout.write("<td>%s</td>" % data['url']) fout.write("<td>%s</td>" % data['title']) fout.write("<td>%s</td>" % data['summary']) fout.write("</tr>") self.datas = [] fout.close() def ouput_end(self, path): ''' 输出HTML结束 :param path: 文件存储路径 :return: ''' fout = codecs.open(path, 'a', encoding='utf-8') fout.write("</table>") fout.write("</body>") fout.write("</html>") fout.close()
NodeManager.py
#!/usr/bin/python # -*- coding: UTF-8 -*- from multiprocessing.managers import BaseManager import time from multiprocessing import Process, Queue from DistributeSpider.ControNode.DataOutPut import DataOutPut from DistributeSpider.ControNode.URLManager import URLManager class NodeManager(object): def start_Manager(self, url_q, result_q): ''' 创建一个分布式管理器 :param url_q: url队列 :param result_q: 结果队列 :return: ''' # 把创建的两个队列注册在网络上,利用register方法,callable参数关联了Queue对象, # 将Queue对象在网络中暴露 BaseManager.register('get_task_queue', callable=lambda: url_q) BaseManager.register('get_result_queue', callable=lambda: result_q) # 绑定端口8001 设置验证口令 相当于对象的初始化 manager = BaseManager(address=('', 8001), authkey='ctj'.encode('utf-8')) # 返回manager对象 return manager def url_manager_proc(self, url_q, conn_q, root_url): url_manager = URLManager() url_manager.add_new_url(root_url) while True: while url_manager.has_new_url(): # 从URL管理器中获取新的url new_url = url_manager.get_new_url() # 将新的URL发送给工作节点 url_q.put(new_url) print('old_url=', url_manager.old_url_size()) # 加一个判断条件,当爬去2000个链接后就关闭,并保存进度 if (url_manager.old_url_size() > 2000): # 通知爬行节点工作结束 url_q.put('end') print('控制节点发起结束通知!') # 关闭管理节点,同时存储set状态 url_manager.save_progress('new_urls.txt', url_manager.new_urls) url_manager.save_progress('old_urls.txt', url_manager.old_urls) return # 将从result_solve_proc获取到的urls添加到URL管理器之间 try: urls = conn_q.get() url_manager.add_new_urls(urls) except BaseException as e: time.sleep(0.1) # 延时休息 def result_solve_proc(self, result_q, conn_q, store_q): while (True): try: if not result_q.empty(): # Queue.get(block=True, timeout=None) content = result_q.get(True) if content['new_urls'] == 'end': # 结果分析进程接受通知然后结束 print('结果分析进程接受通知然后结束!') store_q.put('end') return conn_q.put(content['new_urls']) # url为set类型 store_q.put(content['data']) # 解析出来的数据为dict类型 else: time.sleep(0.1) # 延时休息 except BaseException as e: time.sleep(0.1) # 延时休息 def store_proc(self, store_q): output = DataOutPut() while True: if not store_q.empty(): data = store_q.get() if data == 'end': print('存储进程接受通知然后结束!') output.ouput_end(output.filepath) return output.store_data(data) else: time.sleep(0.1) pass if __name__ == '__main__': # 初始化4个队列 url_q = Queue() result_q = Queue() store_q = Queue() conn_q = Queue() # 创建分布式管理器 node = NodeManager() manager = node.start_Manager(url_q, result_q) # 创建URL管理进程 数据提取进程 数据存储进程 url_manager_proc = Process(target=node.url_manager_proc, args=(url_q, conn_q, 'http://baike.baidu.com/view/284853.htm',)) result_solve_proc = Process(target=node.result_solve_proc, args=(result_q, conn_q, store_q,)) store_proc = Process(target=node.store_proc, args=(store_q,)) # 启动3个进程和分布式管理器 url_manager_proc.start() result_solve_proc.start() store_proc.start() manager.get_server().serve_forever()
URLManager.py
#!/usr/bin/python # -*- coding: UTF-8 -*- import pickle import hashlib class URLManager(object): def __init__(self): self.new_urls = self.load_progress('new_urls.txt') # 未爬取的URl集合 self.old_urls = self.load_progress('old_urls.txt') # 已爬取的URL集合 def has_new_url(self): return self.new_url_size() != 0 def get_new_url(self): new_url = self.new_urls.pop() m = hashlib.md5() m.update(new_url.encode('utf-8')) self.old_urls.add(m.hexdigest()[8:-8]) return new_url def add_new_url(self, url): if url is None: return m = hashlib.md5() m.update(url.encode('utf-8')) url_md5 = m.hexdigest()[8:-8] if url not in self.new_urls and url_md5 not in self.old_urls: self.new_urls.add(url) def add_new_urls(self, urls): if urls is None or len(urls) == 0: for url in urls: self.add_new_url(url) def new_url_size(self): return len(self.new_urls) def old_url_size(self): return len(self.old_urls) def save_progress(self, path, data): with open(path, 'wb') as f: pickle.dump(data, f) def load_progress(self, path): print('[+] 从文件中加载进度:%s' % path) try: with open(path, 'rb') as f: tmp = pickle.load(f) return tmp except: print('[!] 无进度条文件,创建:%s' % path) return set()
HtmlDownloader.py
#!/usr/bin/python # -*- coding: UTF-8 -*- import requests class HtmlDownloader(object): def download(self, url): if url is None: return None user_agent = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)' headers = {'User-Agent': user_agent} r = requests.get(url, headers=headers) if r.status_code == 200: r.encoding = 'utf-8' return r.text return None
SpiderWork.py
#!/usr/bin/python # -*- coding: UTF-8 -*- from multiprocessing.managers import BaseManager from DistributeSpider.SpiderWork.HtmlDownloader import HtmlDownloader from DistributeSpider.SpiderWork.HtmlParser import HtmlParser class SpiderWork(object): def __init__(self): # 初始化分布式进程中的工作节点的连接工作 # 实现第一步:使用BaseManager注册获取Queue的方法名称 BaseManager.register('get_task_queue') BaseManager.register('get_result_queue') # 实现第二步:连接到服务器: server_addr = '127.0.0.1' print(('Connect to server %s...' % server_addr)) # 端口和验证口令注意保持与服务进程设置的完全一致: self.m = BaseManager(address=(server_addr, 8001), authkey='ctj'.encode('utf-8')) # 从网络连接: self.m.connect() # 实现第三步:获取Queue的对象: self.task = self.m.get_task_queue() self.result = self.m.get_result_queue() # 初始化网页下载器和解析器 self.downloader = HtmlDownloader() self.parser = HtmlParser() print('init finish') def crawl(self): while (True): try: if not self.task.empty(): url = self.task.get() if url == 'end': print('控制节点通知爬虫节点停止工作...') # 接着通知其它节点停止工作 self.result.put({'new_urls': 'end', 'data': 'end'}) return print('爬虫节点正在解析:%s' % url.encode('utf-8')) content = self.downloader.download(url) new_urls, data = self.parser.parser(url, content) self.result.put({"new_urls": new_urls, "data": data}) except EOFError as e: print("连接工作节点失败") return except Exception as e: print(e) print('Crawl fali ') if __name__ == "__main__": spider = SpiderWork() spider.crawl()
HtmlParser.py
#!/usr/bin/python # -*- coding: UTF-8 -*- import re import urllib.parse from bs4 import BeautifulSoup class HtmlParser(object): def parser(self, page_url, html_cont): ''' 用于解析网页内容抽取URL和数据 :param page_url: 下载页面的URL :param html_cont: 下载的网页内容 :return:返回URL和数据 ''' if page_url is None or html_cont is None: return soup = BeautifulSoup(html_cont, 'lxml') new_urls = self._get_new_urls(page_url, soup) new_data = self._get_new_data(page_url, soup) return new_urls, new_data def _get_new_urls(self, page_url, soup): ''' 抽取新的URL集合 :param page_url: 下载页面的URL :param soup:soup :return: 返回新的URL集合 ''' new_urls = set() # 抽取符合要求的a标签 links = soup.find_all('a', href=re.compile(r'/item/.*')) for link in links: # 提取href属性 new_url = link['href'] # 拼接成完整网址 new_full_url = urllib.parse.urljoin(page_url, new_url) print(new_full_url) new_urls.add(new_full_url) return new_urls def _get_new_data(self, page_url, soup): ''' 抽取有效数据 :param page_url:下载页面的URL :param soup: :return:返回有效数据 ''' data = {} data['url'] = page_url title = soup.find('dd', class_='lemmaWgt-lemmaTitle-title').find('h1') data['title'] = title.get_text() summary = soup.find('div', class_='lemma-summary') # 获取到tag中包含的所有文版内容包括子孙tag中的内容,并将结果作为Unicode字符串返回 data['summary'] = summary.get_text() return data
项目的源代码下载 https://github.com/tanjunchen/SpiderProject/tree/master/HouseWorldNewHouse