串行爬虫sequentipl

from link_crawler import link_crawler
from mongo_cache import MongoCache
from alexa_cb import AlexaCallback

def main():
    scrape_callback = AlexaCallback()
    cache = MongoCache()
    link_crawler(scrape_callback.seed_url, scrape_callback=scrape_callback, cache=cache)

if __name__ == '__main__':
    main()

注意：报pymongo.errors.OperationFailure: exception: Index with name: timestamp_1 already exists with different options时，需要注掉mongo_cache中的

self.db.webpage.create_index('timestamp', expireAfterSeconds=expires.total_seconds())