python调度器_Python基本爬虫（调度器）

# -*-encoding:utf-8 -*-

from DataOutput import DataOutput

from HtmlDownloader import HtmlDownloader

from HtmlParser import HtmlParse

from URLmanager import UrlManager

class SpiderMan(object):

def __init__(self):

super(SpiderMan, self).__init__()

self.manager = UrlManager()

self.downloader = HtmlDownloader()

self.parser = HtmlParse()

self.output = DataOutput()

def crawl(self,root_url):

#添加入口url

self.manager.add_new_url(root_url)

#判断url管理器中是否有新的url

while(self.manager.has_new_url()and self.manager.old_url_size()<100):

try:

print("1")

#从URL管理器获取新的url

new_url = self.manager.get_new_url()

print("2")

#从HTML下载器下载网页

html = self.downloader.download(new_url)

print("3")

#HTML解析器对网页进行解析

new_urls,data = self.parser.parse(new_url,html)

#将抽取的url添加到url管理器中

print("4")

self.manager.add_new_urls(new_urls)

#数据存储器存储文件

print("5")

#print(data)

self.output.store_data(data)

print("已经抓取%s个链接"%self.manager.old_url_size())

except Exception as e:

print("crawl failed")

self.output.output_html()

if __name__ == '__main__':

Spider_Man = SpiderMan()

Spider_Man.crawl("http://baike.baidu.com/view/284853.html")

由于在解析html页面时没有对获取到的超链接进行很好的提取所以能获取到的数据有限