爬取完成后程序就中断了,也没有输出html,请大家看看是哪里有问题import urllib.request
"""
Html下载器
"""
class HtmlDownloader(object):
@staticmethod
def download(url):
"""
下载该页面
:param url:
:return:
"""
if url is None:
return None
# 打开一个url,返回一个 http.client.HTTPResponse
response = urllib.request.urlopen(url)
# 若请求失败
if response.getcode() != 200:
return None
return response.read()"""
Html输出器
"""
class HtmlOutputer(object):
def __init__(self):
self.datas = []
def collect_data(self, data):
"""
收集数据
:param data:
:return:
"""
if data is None:
return
self.datas.append(data)
def output_html(self):
"""
将收集结果输出成Html页面
:return:
"""
file_out = open('output.html', 'w')
file_out.writelines('')
file_out.writelines('
')file_out.writelines('
for data in self.datas:
file_out.writelines('
')file_out.writelines('
%s' % data['url'])file_out.writelines('
%s' % data['title']. encode('utf-8'))file_out.writelines('
%s' % data['summary']. encode('utf-8'))file_out.writelines('
')file_out.writelines('
')file_out.writelines('')
file_out.writelines('')
file_out.close()#解析器
import re
import urllib.parse
from bs4 import BeautifulSoup
class HtmlParser(object):
def parse(self, page_url, html_content):
"""
解析该页面
:param page_url:
:param html_content:
:return:
"""
if page_url is None or html_content is None:
return
soup = BeautifulSoup(html_content, 'html.parser', from_encoding='utf-8')
new_urls = self._get_new_urls(page_url, soup)
new_data = self._get_new_data(page_url, soup)
return new_urls, new_data
@staticmethod
def _get_new_urls(page_url, soup):
"""
获取该页面中所有的符合检验规则的url
:param page_url:
:param soup:
:return:
"""
# 新的带爬取的url集合
new_urls = set()
# 获取所有符合检验规则的url
links = soup.find_all('a', href=re.compile(r'/item/'))
for link in links:
new_url = link['href']
# 将相对路径的url拼接成绝对路径的url
new_full_url = urllib.parse.urljoin(page_url, new_url)
new_urls.add(new_full_url)
return new_urls
@staticmethod
def _get_new_data(page_url, soup):
"""
整合页面的数据
:param page_url:
:param soup:
:return:
"""
# 该页面整合的数据
res_data = {'url': page_url}
"""
获取爬取页面的标题
Python
"""
title_node = soup.find('dd', class_='lemmaWgt-lemmaTitle-title').find('h1')
res_data['title'] = title_node.get_text()
"""
获取爬取页面的概要
Python语法简洁清晰,特色之一是强制用空白符(white space)作为语句缩进。
"""
summary_node = soup.find('div', class_='lemma-summary')
res_data['summary'] = summary_node.get_text()
return res_datafrom baike_spider import url_manager, html_downloader, html_parser, html_outputer
"""
爬虫主程序
"""
class SpiderMain(object):
# 初始化爬虫
def __init__(self):
# URL管理器
self.urls = url_manager.UrlManager()
# HTML下载器
self.downloader = html_downloader.HtmlDownloader()
# HTML解析器
self.parser = html_parser.HtmlParser()
# HTML输出器
self.outputer = html_outputer.HtmlOutputer()
# 爬虫的调动程序
def craw(self, root_url):
# 记录当前爬取的url序号
cnt = 1
# 添加爬取入口的url
self.urls.add_new_url(root_url)
# 若有新的待爬取的url,则一直循环爬取
while self.urls.has_new_url():
try:
# 获取新的待爬取的url
new_url = self.urls.get_new_url()
# 打印当前爬取的url序号与名字
print('craw %d : %s' % (cnt, new_url))
# 下载爬取的页面
html_content = self.downloader.download(new_url)
# 解析爬取的页面
new_urls, new_data = self.parser.parse(new_url, html_content)
# 添加批量的带爬取的url
self.urls.add_new_urls(new_urls)
# 收集数据
self.outputer.collect_data(new_data)
# 爬取目标定为爬取20个url,若完成任务,则退出循环
if cnt == 10:
break
cnt += 1
except:
print('craw failed.')
# 输出收集好的数据
self.outputer.output_html()
if __name__ == '__main__':
# 爬虫入口页面
root_url = 'http://baike.baidu.com/item/Python'
obj_spider = SpiderMain()
# 启动爬虫
obj_spider.craw(root_url)"""
URL管理器
"""
class UrlManager(object):
def __init__(self):
self.new_urls = set() # 待爬取的url集合
self.old_urls = set() # 已爬取的url集合
def add_new_url(self, url):
"""
添加新的带爬取的url
:param url:
:return:
"""
if url is None:
return
if url not in self.new_urls and url not in self.old_urls:
# 若为新的Url,则添加进待爬取的url集合中
self.new_urls.add(url)
def add_new_urls(self, urls):
"""
批量添加新的带爬取的url集合
:param urls:
:return:
"""
if urls is None or len(urls) == 0:
return
for url in urls:
self.add_new_url(url)
def has_new_url(self):
"""
检测待爬取集合是否为空
:return:
"""
return len(self.new_urls) != 0
def get_new_url(self):
"""
获取要爬取的url
:return:
"""
new_url = self.new_urls.pop() # 获取url并从集合中剔除该url
self.old_urls.add(new_url) # 将获取的url添加进已爬取的url中
return new_url