mongodb 索引去重_mongodb去重 - osc_p9lyobzy的个人空间 - OSCHINA - 中文开源技术交流社区...

frompymongo import DeleteOnefromthreading import Threadfromapscheduler.schedulers.blocking import BlockingSchedulerfromApplication.Utils.Log import LogclassDupKeywordRemove:

def __init__(self):

models=[monde1, monde2, monde2,

monde3, monde4, monde4] # mongoengine的modle

self.pipeline=[

# 根据几个字段去判断键值对的唯一性，这里特别写明了{"$exists": True}，必须保证需要判断的字段完成，否则会影响到后面的group

{"$match": {"keyword": {"$exists": True},"country_id": {"$exists": True},

}},

# 将重复的键值对group起来，并用count计数

{"$group": {"_id": {"keyword": "$keyword","country_id": "$country_id",

},"count": {"$sum": 1}

}},

# 匹配count大于2的键值对，他们就是重复的

{"$match": {"count": {"$gt": 1}

}}

]

self.main(models)

def find_dup_id(self, model):try:

_collection=model._get_collection()

# 配置allowDiskUse=True应对mongodb的16M limit

all_dup_key= list(_collection.aggregate(self.pipeline, allowDiskUse=True))

delete_list=[]

# 重复的doc作为list返回，每一组键值对作为find条件去数据库里查询for dup_event inall_dup_key:

match_pipe= dup_event['_id']

remove_id_list=[]

dups=list(_collection.find(match_pipe))if len(dups) >= 2:for key indups:

remove_id_list.append(key['_id'])

needToSave=remove_id_list.pop() # pop一个出去，作为保留的docfor to_del inremove_id_list:

delete_list.append(

DeleteOne({'_id': to_del

})

)

print(_collection, len(delete_list))ifdelete_list:

print('删除重复数据')

_collection.bulk_write(delete_list)else:

print('无重复数据')

pass

except Exceptionase:

Log('keyword_dup_remove').info(e)

def main(self, models):

t_li=[]for _model inmodels:

t_li.append(

Thread(

target=self.find_dup_id,

kwargs={'model': _model,

}

)

)for t int_li:

t.start()for t int_li:

t.join()if __name__ == '__main__':

DupKeywordRemove()