from link_crawler import link_crawler from mongo_cache import MongoCache from alexa_cb import AlexaCallback def main(): scrape_callback = AlexaCallback() cache = MongoCache() link_crawler(scrape_callback.seed_url, scrape_callback=scrape_callback, cache=cache) if __name__ == '__main__': main()
注意:报pymongo.errors.OperationFailure: exception: Index with name: timestamp_1 already exists with different options时,需要注掉mongo_cache中的
self.db.webpage.create_index('timestamp', expireAfterSeconds=expires.total_seconds())