python MD5 图片去重

最新推荐文章于 2024-06-15 15:23:37 发布

o0stinger0o

最新推荐文章于 2024-06-15 15:23:37 发布

阅读量293

点赞数

文章标签： python 开发语言后端

本文链接：https://blog.csdn.net/qq_36891089/article/details/122519431

版权

import hashlib
import os
import time


def getmd5(filename):
    file_txt = open(filename, 'rb').read()
    myhash = hashlib.md5(file_txt)
    return myhash.hexdigest()





if __name__ == '__main__':
    path = os.getcwd()+"\image"

    all_md5 = {}
    all_size = {}
    total_file=0
    total_delete=0

    start = time.perf_counter()
    for file in os.listdir(path):
        total_file += 1
        real_path=os.path.join(path,file)
        if os.path.isfile(real_path) == True:
            size = os.stat(real_path).st_size
            name_and_md5=[real_path,'']
            if size in all_size.keys():
                new_md5 = getmd5(real_path)
                if all_size[size][1]=='':
                    all_size[size][1]=getmd5(all_size[size][0])
                if new_md5 in all_size[size]:
                    #os.remove(os.path.join(path,file))
                    print ('删除',file)
                    total_delete += 1
                else:
                    all_size[size].append(new_md5)
            else:
                all_size[size]=name_and_md5


    end = time.perf_counter()
    time_last = end - start
    print ('文件总数：', total_file)
    print ('删除个数：', total_delete)
    print ('耗时：', time_last, '秒')