MongoCache

最新推荐文章于 2024-05-20 09:36:45 发布

浮东源

最新推荐文章于 2024-05-20 09:36:45 发布

阅读量207

点赞数 1

分类专栏：爬虫

本文链接：https://blog.csdn.net/qq_33722246/article/details/85106656

版权

爬虫专栏收录该内容

4 篇文章 0 订阅

订阅专栏

# coding=utf-8

import pickle
import zlib
from datetime import datetime,timedelta

import requests
from pymongo import MongoClient
from bson.binary import Binary


class MongoCache(object):
    """
    数据库缓存
    timedela时间间隔
    """
    def __init__(self,client=None,expires=timedelta(days=30)):
        self.client = MongoClient("localhost",27017)
        self.db = self.client.cache
        # 加速查找设置索引，设置超时时间，如果达到expirAfterSeconds设置的超时时间，mongodb会把超时数据自动删除
        self.db.webpage.create_index('timestamp',expireAfterSeconds=expires.total_seconds())


    def __setitem__(self, key, value):
        record = {"result":Binary(zlib.compress(pickle.dumps(value))),"timestamp":datetime.utcnow()}
        # 视同update的uosert(如果不存在执行insert,存在执行update)参数进行插入更新，$set内置函数表示铺盖原始数据
        self.db.webpage.update_one({"_id":key},{'$set':record},upsert=True)

    def __getitem__(self, item):
        # 根据_id以item作为关键字（如http://www.baiu.com)查找相关网页
        record = self.db.webpage.find_one({"_id":item})
        if record:
            return pickle.loads(zlib.decompress(record["result"]))
        else:
            raise KeyError(item + "does not exist")

    def __contains__(self, item):
        try:
            # 调用__getitem__方法
            self[item]
        except KeyError:
            # 捕获到keyerror异常说明没找到相关数，参考33行
            return False
        else:
            # 找到相应数据说明数据库包下载
            return True

    def clear(self):
        self.db.webpage.drop()


if __name__ == '__main__':
    down_url = 'http://www.runoob.com/html/html-tutorial.html'
    m_cache = MongoCache()

    response = requests.get(down_url)
    m_cache[down_url] = response.content
    # print(m_cache[down_url].decode('utf-8'))
    # print(down_url in m_cache)