Dal介绍2：自动分页功能

上一篇文章介绍了Dal的结构和基本方法。本文主要分析缓存+db的分页方法。

find_by_page：分页查询

@ctime(NAME)
    """
    table：mongodb的表名
    prefix：缓存前缀
    query：查询条件，dict类型
    cache：是否使用缓存
    cache_time：缓存时间
    sort：排序条件
    criteria：查询的字段
    page：页数
    count：每页的数量
    cache_kw：关联的缓存key
    pack：缓存value是否使用msgpack
    """
    def find_by_page(self, table, prefix="", query={}, cache_time=43200, sort=None, page=1, count=20, cache_kw=None, criteria=None):
        result = []
        total = 0
        page_count = 0
        current_count = 0
        try:
            sorted_id_result = None
            '''生成以pagecache_table开头的缓存key'''
            key = self.redis_proxy.generateKey(table, prefix, query, sort, name="pagecache", criteria=criteria)
            start, stop = self.get_range_by_page(page, count)

            '''检查缓存的key是否存在'''
            if self.redis_proxy.exists(key):
                '''获取总数'''
                total = self.redis_proxy.strict_zcard(key)

                esc = True
                if sort and len(sort) > 1 and sort[1] < 0:
                    esc = False

                if esc:
                    '''升序获取缓存value'''
                    sorted_id_result = self.redis_proxy.strict_zrange(key, start, stop, prefix)
                else:
                    '''降序获取缓存value'''
                    sorted_id_result = self.redis_proxy.strict_zrevrange(key, start, stop, prefix)
            else:
                '''重新从db加载分页数据'''
                sorted_id_result = self.load_page_data(table, prefix, query, cache_time, sort, cache_kw=cache_kw, criteria=criteria)

                if cache_kw:
                    '''关联缓存'''
                    self.cacheKeyword(key,query,cache_kw)
                total = len(sorted_id_result)

                if count > 0:
                    '''取出主键_id'''
                    sorted_id_result = [r.get("_id") for r in sorted_id_result][start:stop+1]
                else:
                    sorted_id_result = [r.get("_id") for r in sorted_id_result]

            current_count = len(sorted_id_result)
            if self.debug:
                self.logger.debug('[Dal.find_by_page] table=%s,prefix=%s,query=%s,sort=%s,page=%s,count=%s,criteria=%s' %(table,prefix,query,sort,page,count,criteria))

            if not sorted_id_result:
                return result,page_count,current_count,total

            page_count = len(sorted_id_result)

            for _id in sorted_id_result:
                '''根据_id查询记录'''
                item = self.find_one(table, query={"_id":ObjectId(_id)},criteria=criteria,cache=True,cache_time=300)

                '''生成find_one的key'''
                find_one_key=self.redis_proxy.generateKey(table, "find_one",{"_id":_id},criteria=criteria,pack=True)

                '''添加到关联缓存的集合'''
                self.cacheKeyword(find_one_key,{},cache_kw,prefix="")
                if item:
                    result.append(item)

            return result,page_count,current_count,total
        except Exception, e:
            self.logger.error('[Dal.find_by_page] error %s, bt: %s' %(e, traceback.format_exc()))
            return result,page_count,current_count,total

load_page_data：重新加载分页数据

@ctime(NAME)
    def load_page_data(self, table, prefix="", query={}, cache_time=43200, sort=None, cache_kw=None, criteria=None):
        '''生成缓存key'''
        key = self.redis_proxy.generateKey(table, prefix, query, sort, name="pagecache", criteria=criteria, pack=True)
        fields = {"_id":1}
        if sort:
            fields[sort[0]] = 1
        if sort:
            sort_field =sort[0]
        else:
            sort_field = "sort_field"

        '''查询db'''
        cursor = self.get_mongodb()[table].find(query,fields)
        if sort:
            '''排序'''
            if isinstance(sort, tuple):
                cursor = cursor.sort(*sort)
            else:
                cursor = cursor.sort(sort)

        sorted_id_result = list(cursor)
        for r in sorted_id_result:
            r["_id"] = str(r.get("_id"))

            '''获取排序的分数'''
            if sort:
                score = r.get(sort_field, 0.0)
            else:
                score = 0

            if score:
                try:
                    score = float(score)
                except ValueError:
                    score = 0
            r[sort_field] = score

        '''生成排序分数的列表'''
        z_id_score_list = [(r.get("_id"),0.0 if not sort else r.get(sort_field)) for r in sorted_id_result]

        '''添加到redis的sortedset'''
        self.redis_proxy.strict_pipeline_zadd(key, z_id_score_list)

        '''设置缓存时间'''
        self.get_redis().expire(key, cache_time)
        if cache_kw:
            self.cacheKeyword(key,query,cache_kw)
        return sorted_id_result

cacheKeyword：缓存关联的key集合

def cacheKeyword(self,key,query,cache_kw,prefix=""):
        cache_time=86400
        if prefix is None:
            prefix = ""

        '''把cache_kw放入redis的set集合'''
        if cache_kw:

            '''当cache_kw是str、unicode类型'''
            if isinstance(cache_kw,(str,unicode)):
                vkey = query.get(cache_kw) if cache_kw in query else cache_kw
                cache_key = prefix+vkey
                self.redis_proxy.strict_sadd(cache_key,key,pack=False,cache_time=cache_time)

            '''当cache_kw是tuple类型'''
            elif isinstance(cache_kw,tuple):
                for kw in cache_kw:
                    vkey = query.get(cache_kw) if cache_kw in query else cache_kw
                    cache_key = prefix+vkey
                    self.redis_proxy.strict_sadd(cache_key,key,pack=False,cache_time=cache_time)

            '''当cache_kw是dict类型'''
            elif isinstance(cache_kw,dict):
                for key,value in cache_kw.iteritems():
                    cache_key =  "%s%s_%s" %(prefix,key,value)
                    self.redis_proxy.strict_sadd(cache_key,key,pack=False,cache_time=cache_time)