因为硬盘空间有限，最近硬盘快装满了，又发现不少文件是重复的，不想手动的一个一个去对比删除，于是写一个程序来做这件事情。时间不多，没有全部完成，但重要的计算比较重复的功能已经完成。
思路：输入几个目标文件夹，检测到所有重复的文件结果。
文件重复的检测标准有几种方案：
1. MD5值一样，这样对于大量占空间的文件计算来说比较耗费时间，而且可能存在碰撞情况，可以再加上文件名相同可修改时间相同的条件，目前是以MD5值为键值来保存所有计算完的文件数据。
2. 可以使用文件的属性，包括文件名，文件大小，修改时间；这样的话，可以基本检测到重复文件，而且消耗时间应该比计算MD5要少，目前没有使用这种方式。
后续需要使用第二种方案与第一种对比，从时间和效果上做出数据对比。接着做一个界面方便操作，确认删除对应的重复文件。
以下代码仅指定一个文件夹计算所有文件的MD5值。
#include "stdafx.h"
#include <stdio.h>
#include <stdlib.h>
#include <map>
#include <list>
#include <direct.h>
#include <Shlwapi.h>
#include <windows.h>
#include "md5.h"
typedef std::multimap<INT64,WIN32_FIND_DATA> map_type;
typedef std::list<std::wstring> list_type;
union union_MD5
{
INT64 i64_md5;
unsigned char c_md5[16];
};
typedef struct MIX_WIN32_FIND_DATA
{
std::wstring fullpath;
WIN32_FIND_DATA wfd;
}MIX_WIN32_FIND_DATA;
DWORD RecurseGetFileList(std::wstring &fullpath,map_type &map_data)
{
DWORD count = 0;
WIN32_FIND_DATA fd;
HANDLE hFile = ::FindFirstFile(fullpath.c_str(),&fd);
if ( INVALID_HANDLE_VALUE == hFile )
{
return 0;
}
do
{
if ( ( 0 != _tcscmp(fd.cFileName,_T(".")))
&& ( 0 != _tcscmp(fd.cFileName,_T("..")) ) )
{
if ( fd.dwFileAttributes == INVALID_FILE_ATTRIBUTES )
{
return 0;
}
if ( FILE_ATTRIBUTE_DIRECTORY & fd.dwFileAttributes )
{
std::wstring newdir = fullpath.substr(0,fullpath.length()-2);
newdir.append(_T("\\")).append(fd.cFileName).append(_T("\\*"));
count += RecurseGetFileList(newdir,map_data);
}
else
{
union_MD5 uMd5;
std::wstring filepath = fullpath.substr(0,fullpath.length()-2);
filepath.append(_T("\\")).append(fd.cFileName);
BOOL bRet = MD5File(filepath.c_str(), uMd5.c_md5);
map_data.insert(std::make_pair(uMd5.i64_md5,fd));
int lowpart = uMd5.i64_md5>>32;
int highpart = uMd5.i64_md5&0xFFFFFFFF;
_tprintf(_T("MD5:%04x %04x filepath: %s\n"),highpart,lowpart,filepath.c_str());
count++;
}
}
if (::FindNextFile(hFile,&fd) == FALSE)
{
break;
}
}while(1);
return count;
}
DWORD GetMultimapForDirList(list_type dirlist,map_type &out_map)
{
DWORD count = 0;
WIN32_FILE_ATTRIBUTE_DATA fad;
list_type::iterator it = dirlist.begin();
while (it!=dirlist.end())
{
count += RecurseGetFileList(*it,out_map);
}
return count;
}
int main(int argc,char ** argv)
{
list_type find_list;
map_type find_map;
find_list.push_front(std::wstring(_T("D:\\SetupFiles\\*")));
GetMultimapForDirList(find_list,find_map);
getchar();
return 0;
}作者是：Copyright (C) 2000 by Robert Hubley.
#if !defined(_LGY_MD5_H)
#define _LGY_MD5_H
/* MD5 Class. */
class MD5_CTX {
public:
MD5_CTX();
virtual ~MD5_CTX();
void MD5Update ( unsigned char *input, unsigned int inputLen);
void MD5Final (unsigned char digest[16]);
private:
unsigned long int state[4]; /* state (ABCD) */
unsigned long int count[2]; /* number of bits, modulo 2^64 (lsb first) */
unsigned char buffer[64]; /* input buffer */
unsigned char PADDING[64]; /* What? */
private:
void MD5Init ();
void MD5Transform (unsigned long int state[4], unsigned char block[64]);
void MD5_memcpy (unsigned char* output, unsigned char* input,unsigned int len);
void Encode (unsigned char *output, unsigned long int *input,unsigned int len);
void Decode (unsigned long int *output, unsigned char *input, unsigned int len);
void MD5_memset (unsigned char* output,int value,unsigned int len);
};
#define MD5_BLEN 16
#define MD5_SLEN 32
#define NULL_MD5 "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
#define MD5_ISNULL(MD5BIN) (memcmp((const void *)NULL_MD5,(const void *)MD5BIN,MD5_BLEN)==0)
BOOL MD5Data(unsigned char *buff,unsigned int len, unsigned char digest[16]);
BOOL MD5File(LPCTSTR lpszFileName, unsigned char digest[16], unsigned int * pFileLen=NULL);
#endif#include "stdafx.h"
#include "md5.h"
/* Constants for MD5Transform routine.
*/
#define S11 7
#define S12 12
#define S13 17
#define S14 22
#define S21 5
#define S22 9
#define S23 14
#define S24 20
#define S31 4
#define S32 11
#define S33 16
#define S34 23
#define S41 6
#define S42 10
#define S43 15
#define S44 21
/* F, G, H and I are basic MD5 functions.
*/
#define F(x, y, z) (((x) & (y)) | ((~x) & (z)))
#define G(x, y, z) (((x) & (z)) | ((y) & (~z)))
#define H(x, y, z) ((x) ^ (y) ^ (z))
#define I(x, y, z) ((y) ^ ((x) | (~z)))
/* ROTATE_LEFT rotates x left n bits.
*/
#define ROTATE_LEFT(x, n) (((x) << (n)) | ((x) >> (32-(n))))
/* FF, GG, HH, and II transformations for rounds 1, 2, 3, and 4.
Rotation is separate from addition to prevent recomputation.
*/
#define FF(a, b, c, d, x, s, ac) { \
(a) += F ((b), (c), (d)) + (x) + (unsigned long int)(ac); \
(a) = ROTATE_LEFT ((a), (s)); \
(a) += (b); \
}
#define GG(a, b, c, d, x, s, ac) { \
(a) += G ((b), (c), (d)) + (x) + (unsigned long int)(ac); \
(a) = ROTATE_LEFT ((a), (s)); \
(a) += (b); \
}
#define HH(a, b, c, d, x, s, ac) { \
(a) += H ((b), (c), (d)) + (x) + (unsigned long int)(ac); \
(a) = ROTATE_LEFT ((a), (s)); \
(a) += (b); \
}
#define II(a, b, c, d, x, s, ac) { \
(a) += I ((b), (c), (d)) + (x) + (unsigned long int)(ac); \
(a) = ROTATE_LEFT ((a), (s)); \
(a) += (b); \
}
/* MD5 initialization. Begins an MD5 operation, writing a new context.
*/
MD5_CTX::MD5_CTX()
{
MD5Init ();
}
MD5_CTX::~MD5_CTX()
{
}
void MD5_CTX::MD5Init ()
{
this->count[0] = this->count[1] = 0;
/* Load magic initialization constants.*/
this->state[0] = 0x67452301;
this->state[1] = 0xefcdab89;
this->state[2] = 0x98badcfe;
this->state[3] = 0x10325476;
/* Add by Liguangyi */
MD5_memset(PADDING, 0, sizeof(PADDING));
*PADDING=0x80;
//PADDING = {
// 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
// 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
// 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
}
/* MD5 block update operation. Continues an MD5 message-digest
operation, processing another message block, and updating the
context.
*/
void MD5_CTX::MD5Update (unsigned char *input,unsigned int inputLen)
{
unsigned int i, index, partLen;
/* Compute number of bytes mod 64 */
index = (unsigned int)((this->count[0] >> 3) & 0x3F);
/* Update number of bits */
if ((this->count[0] += ((unsigned long int)inputLen << 3))
< ((unsigned long int)inputLen << 3))
this->count[1]++;
this->count[1] += ((unsigned long int)inputLen >> 29);
partLen = 64 - index;
/* Transform as many times as possible.
*/
if (inputLen >= partLen) {
MD5_memcpy((unsigned char*)&this->buffer[index],
(unsigned char*)input, partLen);
MD5Transform (this->state, this->buffer);
for (i = partLen; i + 63 < inputLen; i += 64)
MD5Transform (this->state, &input[i]);
index = 0;
}
else
i = 0;
/* Buffer remaining input */
MD5_memcpy ((unsigned char*)&this->buffer[index], (unsigned char*)&input[i], inputLen-i);
}
/* MD5 finalization. Ends an MD5 message-digest operation, writing the
the message digest and zeroizing the context.
*/
void MD5_CTX::MD5Final (unsigned char digest[16])
{
unsigned char bits[8];
unsigned int index, padLen;
/* Save number of bits */
Encode (bits, this->count, 8);
/* Pad out to 56 mod 64.
*/
index = (unsigned int)((this->count[0] >> 3) & 0x3f);
padLen = (index < 56) ? (56 - index) : (120 - index);
MD5Update ( PADDING, padLen);
/* Append length (before padding) */
MD5Update (bits, 8);
/* Store state in digest */
Encode (digest, this->state, 16);
/* Zeroize sensitive information.
*/
MD5_memset ((unsigned char*)this, 0, sizeof (*this));
this->MD5Init();
}
/* MD5 basic transformation. Transforms state based on block.
*/
void MD5_CTX::MD5Transform (unsigned long int state[4], unsigned char block[64])
{
unsigned long int a = state[0], b = state[1], c = state[2], d = state[3], x[16];
Decode (x, block, 64);
/* Round 1 */
FF (a, b, c, d, x[ 0], S11, 0xd76aa478); /* 1 */
FF (d, a, b, c, x[ 1], S12, 0xe8c7b756); /* 2 */
FF (c, d, a, b, x[ 2], S13, 0x242070db); /* 3 */
FF (b, c, d, a, x[ 3], S14, 0xc1bdceee); /* 4 */
FF (a, b, c, d, x[ 4], S11, 0xf57c0faf); /* 5 */
FF (d, a, b, c, x[ 5], S12, 0x4787c62a); /* 6 */
FF (c, d, a, b, x[ 6], S13, 0xa8304613); /* 7 */
FF (b, c, d, a, x[ 7], S14, 0xfd469501); /* 8 */
FF (a, b, c, d, x[ 8], S11, 0x698098d8); /* 9 */
FF (d, a, b, c, x[ 9], S12, 0x8b44f7af); /* 10 */
FF (c, d, a, b, x[10], S13, 0xffff5bb1); /* 11 */
FF (b, c, d, a, x[11], S14, 0x895cd7be); /* 12 */
FF (a, b, c, d, x[12], S11, 0x6b901122); /* 13 */
FF (d, a, b, c, x[13], S12, 0xfd987193); /* 14 */
FF (c, d, a, b, x[14], S13, 0xa679438e); /* 15 */
FF (b, c, d, a, x[15], S14, 0x49b40821); /* 16 */
/* Round 2 */
GG (a, b, c, d, x[ 1], S21, 0xf61e2562); /* 17 */
GG (d, a, b, c, x[ 6], S22, 0xc040b340); /* 18 */
GG (c, d, a, b, x[11], S23, 0x265e5a51); /* 19 */
GG (b, c, d, a, x[ 0], S24, 0xe9b6c7aa); /* 20 */
GG (a, b, c, d, x[ 5], S21, 0xd62f105d); /* 21 */
GG (d, a, b, c, x[10], S22, 0x2441453); /* 22 */
GG (c, d, a, b, x[15], S23, 0xd8a1e681); /* 23 */
GG (b, c, d, a, x[ 4], S24, 0xe7d3fbc8); /* 24 */
GG (a, b, c, d, x[ 9], S21, 0x21e1cde6); /* 25 */
GG (d, a, b, c, x[14], S22, 0xc33707d6); /* 26 */
GG (c, d, a, b, x[ 3], S23, 0xf4d50d87); /* 27 */
GG (b, c, d, a, x[ 8], S24, 0x455a14ed); /* 28 */
GG (a, b, c, d, x[13], S21, 0xa9e3e905); /* 29 */
GG (d, a, b, c, x[ 2], S22, 0xfcefa3f8); /* 30 */
GG (c, d, a, b, x[ 7], S23, 0x676f02d9); /* 31 */
GG (b, c, d, a, x[12], S24, 0x8d2a4c8a); /* 32 */
/* Round 3 */
HH (a, b, c, d, x[ 5], S31, 0xfffa3942); /* 33 */
HH (d, a, b, c, x[ 8], S32, 0x8771f681); /* 34 */
HH (c, d, a, b, x[11], S33, 0x6d9d6122); /* 35 */
HH (b, c, d, a, x[14], S34, 0xfde5380c); /* 36 */
HH (a, b, c, d, x[ 1], S31, 0xa4beea44); /* 37 */
HH (d, a, b, c, x[ 4], S32, 0x4bdecfa9); /* 38 */
HH (c, d, a, b, x[ 7], S33, 0xf6bb4b60); /* 39 */
HH (b, c, d, a, x[10], S34, 0xbebfbc70); /* 40 */
HH (a, b, c, d, x[13], S31, 0x289b7ec6); /* 41 */
HH (d, a, b, c, x[ 0], S32, 0xeaa127fa); /* 42 */
HH (c, d, a, b, x[ 3], S33, 0xd4ef3085); /* 43 */
HH (b, c, d, a, x[ 6], S34, 0x4881d05); /* 44 */
HH (a, b, c, d, x[ 9], S31, 0xd9d4d039); /* 45 */
HH (d, a, b, c, x[12], S32, 0xe6db99e5); /* 46 */
HH (c, d, a, b, x[15], S33, 0x1fa27cf8); /* 47 */
HH (b, c, d, a, x[ 2], S34, 0xc4ac5665); /* 48 */
/* Round 4 */
II (a, b, c, d, x[ 0], S41, 0xf4292244); /* 49 */
II (d, a, b, c, x[ 7], S42, 0x432aff97); /* 50 */
II (c, d, a, b, x[14], S43, 0xab9423a7); /* 51 */
II (b, c, d, a, x[ 5], S44, 0xfc93a039); /* 52 */
II (a, b, c, d, x[12], S41, 0x655b59c3); /* 53 */
II (d, a, b, c, x[ 3], S42, 0x8f0ccc92); /* 54 */
II (c, d, a, b, x[10], S43, 0xffeff47d); /* 55 */
II (b, c, d, a, x[ 1], S44, 0x85845dd1); /* 56 */
II (a, b, c, d, x[ 8], S41, 0x6fa87e4f); /* 57 */
II (d, a, b, c, x[15], S42, 0xfe2ce6e0); /* 58 */
II (c, d, a, b, x[ 6], S43, 0xa3014314); /* 59 */
II (b, c, d, a, x[13], S44, 0x4e0811a1); /* 60 */
II (a, b, c, d, x[ 4], S41, 0xf7537e82); /* 61 */
II (d, a, b, c, x[11], S42, 0xbd3af235); /* 62 */
II (c, d, a, b, x[ 2], S43, 0x2ad7d2bb); /* 63 */
II (b, c, d, a, x[ 9], S44, 0xeb86d391); /* 64 */
state[0] += a;
state[1] += b;
state[2] += c;
state[3] += d;
/* Zeroize sensitive information.
*/
MD5_memset ((unsigned char*)x, 0, sizeof (x));
}
/* Encodes input (unsigned long int) into output (unsigned char). Assumes len is
a multiple of 4.
*/
void MD5_CTX::Encode (unsigned char *output, unsigned long int *input,unsigned int len)
{
unsigned int i, j;
for (i = 0, j = 0; j < len; i++, j += 4) {
output[j] = (unsigned char)(input[i] & 0xff);
output[j+1] = (unsigned char)((input[i] >> 8) & 0xff);
output[j+2] = (unsigned char)((input[i] >> 16) & 0xff);
output[j+3] = (unsigned char)((input[i] >> 24) & 0xff);
}
}
/* Decodes input (unsigned char) into output (unsigned long int). Assumes len is
a multiple of 4.
*/
void MD5_CTX::Decode (unsigned long int *output, unsigned char *input, unsigned int len)
{
unsigned int i, j;
for (i = 0, j = 0; j < len; i++, j += 4)
output[i] = ((unsigned long int)input[j]) | (((unsigned long int)input[j+1]) << 8) |
(((unsigned long int)input[j+2]) << 16) | (((unsigned long int)input[j+3]) << 24);
}
/* Note: Replace "for loop" with standard memcpy if possible.
*/
void MD5_CTX::MD5_memcpy (unsigned char* output, unsigned char* input,unsigned int len)
{
unsigned int i;
for (i = 0; i < len; i++)
output[i] = input[i];
}
/* Note: Replace "for loop" with standard memset if possible.
*/
void MD5_CTX::MD5_memset (unsigned char* output,int value,unsigned int len)
{
unsigned int i;
for (i = 0; i < len; i++)
((char *)output)[i] = (char)value;
}
BOOL MD5Data(unsigned char *buff,unsigned int len, unsigned char digest[16])
{
MD5_CTX md5;
md5.MD5Update ((unsigned char *)buff, len);
md5.MD5Final (digest);
return TRUE;
}
BOOL MD5File(LPCTSTR lpszFileName, unsigned char digest[16], unsigned int * pFileLen)
{
BOOL bRet=FALSE;
memset(digest,0,sizeof(digest));
HANDLE hFile = CreateFile(lpszFileName, GENERIC_READ , FILE_SHARE_READ | FILE_SHARE_WRITE , NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
if (hFile != INVALID_HANDLE_VALUE)
{
DWORD dwFileSizeHigh;
DWORD dwFileSizeLow = GetFileSize(hFile, &dwFileSizeHigh);
if (dwFileSizeLow)
{
if (pFileLen) *pFileLen=dwFileSizeLow;
SECURITY_DESCRIPTOR secutityDese;
::InitializeSecurityDescriptor(&secutityDese, SECURITY_DESCRIPTOR_REVISION);
::SetSecurityDescriptorDacl(&secutityDese,TRUE,NULL,FALSE);
SECURITY_ATTRIBUTES securityAttr;
securityAttr.nLength = sizeof SECURITY_ATTRIBUTES;
securityAttr.bInheritHandle = FALSE;
securityAttr.lpSecurityDescriptor = &secutityDese;
HANDLE hFileMap = CreateFileMapping(hFile, &securityAttr, PAGE_READONLY, dwFileSizeHigh, dwFileSizeLow, NULL);
if (hFileMap)
{
LPBYTE lpbMapAddress = (LPBYTE)MapViewOfFile(hFileMap, FILE_MAP_READ, 0, 0, 0);
if (lpbMapAddress)
{
bRet=MD5Data((unsigned char *)lpbMapAddress, dwFileSizeLow, digest);
UnmapViewOfFile(lpbMapAddress);
}
CloseHandle(hFileMap);
}
}
CloseHandle(hFile);
}
return bRet;
}
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
