数据库去重

from collections import defaultdict

def get_hash(name):
all_list = []
data = database.find({‘name’: name}, COLLECTION, BASE, p={‘url’: True})
for url in data:
all_list.append(url[‘images_url’])
return all_list

def list_duplicates(all_list):
tally = defaultdict(list)
for i, item in enumerate(all_list):
tally[item].append(i)
return ((key, locs) for key, locs in tally.items()
if len(locs) > 1)

def delete_data():
‘’’
删除重复数据
:return:
‘’’
company_list = get_company()
for company in company_list:
company_name = company[‘name’]
pop_list = []
all_list = get_hash(company_name)
for dup in sorted(list_duplicates(all_list)):
pop_list.append(dup)
for url in pop_list:
data =database.find({‘company_name’: company_name, ‘url’: url[0]}, COLLECTION, BASE)
for i in data[1::]:
print(i[‘updated_at’])
database.delete({‘updated_at’: i[‘updated_at’], ‘company_name’: company_name}, COLLECTION,
BASE)
if not company_list:
return ‘delete completed’

def get_company():
获取数据库数据
where = {}
name =database.find(where, COLLECTION, BASE, p={‘name’: True})
return name