检测重复文件功能实现(改进版本1)

针对大文件md5计算时间偏长问题，觉得不需要完全计算整个文件的md5也可以初步确定文件是否相同。换作计算一些属性的md5值，极大提高速度，也可以满足需求：

1. 文件名

2. 文件修改时间——根据实验，两个重复文件如果仅仅是复制，创建时间和最后访问时间都会改变，但最后修改时间在内容没有修改时不会是变的。这是根据文件系统特征决定的。

3. 文件大小

以上三点使用以下结构来表示。

typedef struct FileInfoForCompare
{
	DWORD dwFileAttributes;
	FILETIME ftLastWriteTime;
	DWORD nFileSizeHigh;
	DWORD nFileSizeLow;
	WCHAR cFileName[ 260 ];
}FileInfoForCompare_;

以下版本更改计算md5的数据，更正了GetMultimapForDirList函数的while循环没有把迭代器加1的bug。改进之后的代码如下：

#include <windows.h>
#include "md5.h"

typedef struct MIX_WIN32_FIND_DATA
{
	std::wstring fullpath;
	WIN32_FIND_DATA wfd;
}MIX_WIN32_FIND_DATA_;

typedef std::multimap<INT64,MIX_WIN32_FIND_DATA> map_type;
typedef std::list<std::wstring> list_type;
union union_MD5
{
	INT64 i64_md5;
	unsigned char c_md5[16];
};

typedef struct FileInfoForCompare
{
	DWORD dwFileAttributes;
	FILETIME ftLastWriteTime;
	DWORD nFileSizeHigh;
	DWORD nFileSizeLow;
	WCHAR cFileName[ 260 ];
}FileInfoForCompare_;


void GetMd5(MIX_WIN32_FIND_DATA &mwfd,union_MD5 &uMd5)
{
	// use file data md5.
	//MD5File(mwfd.fullpath.c_str(), uMd5.c_md5);

	// use file attribute md5
	FileInfoForCompare fic;
	fic.dwFileAttributes = mwfd.wfd.dwFileAttributes;
	fic.ftLastWriteTime = mwfd.wfd.ftLastWriteTime;
	fic.nFileSizeHigh = mwfd.wfd.nFileSizeHigh;
	fic.nFileSizeLow = mwfd.wfd.nFileSizeLow;
	_tcscpy_s(fic.cFileName,MAX_PATH,mwfd.wfd.cFileName);
	MD5Data((unsigned char*)(&fic),sizeof(fic),uMd5.c_md5);
}

DWORD RecurseGetFileList(std::wstring &fullpath,map_type &map_data)
{
	DWORD count = 0;
	MIX_WIN32_FIND_DATA mwfd;
	WIN32_FIND_DATA &fd = mwfd.wfd;
	HANDLE hFile = ::FindFirstFile(fullpath.c_str(),&fd);
	if ( INVALID_HANDLE_VALUE == hFile )
	{
		return 0;
	}
	do
	{		
		if ( ( 0 != _tcscmp(fd.cFileName,_T(".")))
			&& ( 0 != _tcscmp(fd.cFileName,_T("..")) ) )
		{
			if ( fd.dwFileAttributes == INVALID_FILE_ATTRIBUTES )
			{
				return 0;
			}
			if ( FILE_ATTRIBUTE_DIRECTORY & fd.dwFileAttributes )
			{
				std::wstring newdir = fullpath.substr(0,fullpath.length()-2);
				newdir.append(_T("\\")).append(fd.cFileName).append(_T("\\*"));
				count += RecurseGetFileList(newdir,map_data);
			}
			else
			{	
				union_MD5 uMd5;
				mwfd.fullpath = fullpath.substr(0,fullpath.length()-2);
				mwfd.fullpath.append(_T("\\")).append(fd.cFileName);

				GetMd5(mwfd,uMd5);

				map_data.insert(std::make_pair(uMd5.i64_md5,mwfd));
				int lowpart = uMd5.i64_md5>>32;  
				int highpart = uMd5.i64_md5&0xFFFFFFFF;  
				_tprintf(_T("MD5:%04x %04x filepath: %s\n"),highpart,lowpart,mwfd.fullpath.c_str());
				count++;
			}			
		}

		if (::FindNextFile(hFile,&fd) == FALSE)
		{
			break;
		}

	}while(1);

	return count;
}


DWORD GetMultimapForDirList(list_type dirlist,map_type &out_map)
{
	DWORD count = 0;
	WIN32_FILE_ATTRIBUTE_DATA fad;
	list_type::iterator it = dirlist.begin();
	while (it!=dirlist.end())
	{
		count += RecurseGetFileList(*it,out_map);
		it++;
	}
	return count;
}

int main(int argc,char ** argv)
{
	list_type find_list;
	map_type find_map;
	find_list.push_front(std::wstring(_T("D:\\Setup Files\\*")));
	GetMultimapForDirList(find_list,find_map);
	return 0;
}