下载了些图片,里面有少量重复的。写了个脚本,给每个文件算SHA-256,哈希值重复的就删掉。
import re, sys, os, hashlib
folder = 'C:\\XXX\\';
dicFile = {}
n = 0;
for filename in sorted(os.listdir(folder)):
path = folder + filename;
sig = hashlib.sha256(open(path, 'rb').read()).hexdigest()
if(sig not in dicFile):
dicFile[sig] = filename;
else:
print(filename + " is dupe with " + dicFile[sig]);
os.remove(path);
n = n + 1;
print("total deleted: " + str(n));