自己挖的坑，说什么也要填上。
调试过程：
- 新建一个文件夹
- 新建五个txt文件
- 分别命名，并把 .tex 的文件后缀更改成 .py
- 运行spider_main.py
刚在网上找了代码，自己改写了download和parser两个部分，其他部分些许改动。编译调试的时候，不报错，但是，就是不出结果，也不清楚是什么意思。
运行结果不出错，但是就是没有预期的结果。
现在将代码贴出来，希望大家能集思广益：
1.这是网页管理模块 url_manager.py
#!/usr/bin/python # coding=utf-8 class UrlManager(object): """docstring for UrlManager""" def __init__(self): self.new_urls = set() self.old_urls = set() def add_new_url(self, url): if url is None: return if url not in self.new_urls and url not in self.old_urls: self.new_urls.add(url) def add_new_urls(self, urls): if urls is None or len(urls) == 0: return for url in urls: self.add_new_url(url) def has_new_url(self): return len(self.new_urls) != 0 def get_new_url(self): #pop, 从set中拿一个并移除 new_url = self.new_urls.pop() self.old_urls.add(new_url) return new_url
2.这是下载网页模块 html_downloader.py
#!/usr/bin/python # coding=utf-8 #import urllib2 import urllib.request class HtmlDownloader(object): """docstring for HtmlDownloader""" def __init__(self): pass def download(self, url): if url is None: return None headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) ' 'Chrome/51.0.2704.63 Safari/537.36' } req = urllib.request.Request(url=url, headers=headers) response = urllib.request.urlopen(req) #print response.getcode() if response.getcode() != 200: return None return response.read()
3.这是网页解析模块 html_parser.py
#!/usr/bin/python # coding=utf-8 from bs4 import BeautifulSoup import re import urllib class HtmlParser(object): """docstring for HtmlParser""" def __init__(self): pass def parser(self, page_url, html_cont): #print page_url if page_url is None or html_cont is None: return soup = BeautifulSoup(html_cont, 'html.parser', encoding = 'utf-8') new_urls = self._get_new_urls(page_url, soup) new_data = self._get_new_data(page_url, soup) return new_urls, new_data def _get_new_urls(self, page_url, soup): # /view/Êý×Ö.htm # such as /view/123.htm new_urls = set() links = soup.find_all('a', href = re.compile(r"/view/\d+\.htm")) for link in links: new_url = link['href'] #print new_url #°²×°page_url µÄ¸ñʽ£¬°Ñnew_url²¹È« new_full_url = urlparse.urljoin(page_url, new_url) new_urls.add(new_full_url) #print new_full_url return new_urls def _get_new_data(self, page_url, soup): res_data = {} #×Öµä # url res_data['url'] = page_url #<dd class="lemmaWgt-lemmaTitle-title"> #<h1>Python</h1> title_node = soup.find('dd', class_= "lemmaWgt-lemmaTitle-title").find("h1") res_data['title'] = title_node.get_text() #<div class="lemma-summary" label-module="lemmaSummary"> summary_node = soup.find('div', class_= "lemma-summary") res_data['summary'] = summary_node.get_text() return res_data
4.下面是网页输出模块 html_outputer.py
#!/usr/bin/python # coding=utf-8 import re import urllib import time import urllib.request import requests from bs4 import BeautifulSoup class HtmlOutputer(object): """docstring for HtmlOutputer""" def __init__(self): self.datas = [] def collect_data(self, data): if data is None: return self.datas.append(data) def output_html(self): fout = open('output.html', 'w') fout.write("<html>") fout.write("<head>") # charset 编码 fout.write('<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>') fout.write("</head>") fout.write("<body>") fout.write("<table>") # python 默认 ascii for data in self.datas: fout.write("<tr>") fout.write("<td>%s</td>" % data['url']) fout.write("<td>%s</td>" % data['title'].encode('utf-8')) fout.write("<td>%s</td>" % data['summary'].encode('utf-8')) fout.write("</tr>") fout.write("</table>") fout.write("</body>") fout.write("</html>") fout.close()
5.主函数，运行这个就可以了
#!/usr/bin/python # coding=utf-8 #from baike_spider import url_manager,html_downloader,html_parser,html_outputer import url_manager,html_downloader,html_parser,html_outputer class SpiderMain(object): """docstring for SpiderMain""" def __init__(self): self.urls = url_manager.UrlManager() self.downloader = html_downloader.HtmlDownloader() self.parser = html_parser.HtmlParser() self.outputer = html_outputer.HtmlOutputer() def craw(self,root_url): count = 1 self.urls.add_new_url(root_url) while self.urls.has_new_url(): try: new_url = self.urls.get_new_url() print ('craw %d : %s' % (count, new_url)) html_cont = self.downloader.download(new_url) new_urls, new_data = self.parser.parser(new_url, html_cont) self.urls.add_new_urls(new_urls) self.outputer.collect_data(new_data) if count == 1000: break count = count + 1 except: print ('craw failed') self.outputer.output_html() if __name__ == "__main__": root_url = "http://baike.baidu.com/view/21087.htm" obj_spider = SpiderMain() obj_spider.craw(root_url)
希望大家帮忙一块看，问题在哪里？
3943
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
