/**
* 重复文件管理器
*/
public class RepeatFileManager {
private static final String EMPTY_FILE = "empty_file"; //空文件
private Map<Long, String> mFileSizeMap = new HashMap<>(); //保存文件大小
private Map<String, String> mFileHashMap = new HashMap<>(); //保存文件hash
private Map<String, List<String>> mRepeatFileMap = new HashMap<>(); //最终重复文件
private boolean mCanceled;
private OnRepeatStatsListener onRepeatStatsListener;
public void start(final File dir) {
new Thread() {
public void run() {
mCanceled = false;
mFileSizeMap.clear();
mFileHashMap.clear();
mRepeatFileMap.clear();
calc(dir);
if (onRepeatStatsListener != null) {
onRepeatStatsListener.onRepeatStatsFinished(mRepeatFileMap);
}
};
}.start();
}
private void calc(File dir) {
File[] files = dir.listFiles();
if (files != null && files.length > 0) {
for (File file : files) {
if (file.isDirectory()) {
calc(file);
continue;
}
if (file.length() == 0) {
//空文件
List<String> pathList = mRepeatFileMap.get(EMPTY_FILE);
if (pathList == null) {
pathList = new ArrayList<>();
mRepeatFileMap.put(EMPTY_FILE, pathList);
}
pathList.add(file.getAbsolutePath());
} else {
if(mFileSizeMap.containsKey(file.length())){
//大小重复,计算文件散列
String path = mFileSizeMap.get(file.length());
if (!mFileHashMap.values().contains(path)) {
String firstHash = calcFileKey(new File(path));
if (firstHash != null) {
mFileHashMap.put(firstHash, path);
}
}
String hash = calcFileKey(file);
if (hash != null) {
if (mFileHashMap.containsKey(hash)) {
//散列重复,保存文件路径
List<String> pathList = mRepeatFileMap.get(hash);
if (pathList == null) {
pathList = new ArrayList<>();
pathList.add(mFileHashMap.get(hash));
mRepeatFileMap.put(hash, pathList);
}
pathList.add(file.getAbsolutePath());
} else {
mFileHashMap.put(hash, file.getAbsolutePath());
}
}
}else {
mFileSizeMap.put(file.length(), file.getAbsolutePath());
}
}
if (mCanceled) {
break;
}
}
}
}
/**
* 计算文件唯一散列值
* @param file
* @return
*/
private static String calcFileKey(File file) {
String md5 = calcFileMd5(file);
String crc32 = calcFileCRC32(file);
if (!md5.isEmpty() && !crc32.isEmpty()) {
return md5 + "_" + crc32;
} else {
return null;
}
}
/**
* 获取文件md5
*
* @param file
* @return
*/
private static String calcFileMd5(File file) {
if (!file.isFile()) {
return "";
}
MessageDigest digest = null;
FileInputStream in = null;
byte buffer[] = new byte[1024];
int len;
try {
digest = MessageDigest.getInstance("MD5");
in = new FileInputStream(file);
while ((len = in.read(buffer, 0, 1024)) != -1) {
digest.update(buffer, 0, len);
}
in.close();
return byteToHex(digest.digest());
} catch (Exception e) {
e.printStackTrace();
}
return "";
}
private static String byteToHex(byte[] data) {
if (data != null) {
StringBuilder builder = new StringBuilder();
for (int i = 0; i < data.length; i++) {
int v = data[i] & 0xFF;
builder.append(String.format("%02x", v));
}
return builder.toString();
} else {
return null;
}
}
/**
* 获取文件crc32
* @param file
* @return
*/
private static String calcFileCRC32(File file) {
BufferedInputStream in = null;
try {
in = new BufferedInputStream(new FileInputStream(file));
CRC32 crc = new CRC32();
byte[] bytes = new byte[1024];
int cnt;
while ((cnt = in.read(bytes)) != -1) {
crc.update(bytes, 0, cnt);
}
return Long.toHexString(crc.getValue());
} catch (Exception e) {
e.printStackTrace();
} finally {
try {
if (in != null) {
in.close();
}
} catch (IOException e) {
e.printStackTrace();
}
}
return "";
}
public void testCalc(File file) {
System.out.println(calcFileMd5(file));
}
public Map<String, List<String>> getRepeatFiles() {
return mRepeatFileMap;
}
public void setOnRepeatStatsListener(
OnRepeatStatsListener onRepeatStatsListener) {
this.onRepeatStatsListener = onRepeatStatsListener;
}
public interface OnRepeatStatsListener {
void onRepeatStatsFinished(Map<String, List<String>> repeatFiles);
}
}
public class RepeatFileDemo {
public static void main(String[] args) {
RepeatFileManager rfm = new RepeatFileManager();
final long startTime = System.currentTimeMillis();
rfm.setOnRepeatStatsListener(new RepeatFileManager.OnRepeatStatsListener() {
@Override
public void onRepeatStatsFinished(
Map<String, List<String>> repeatFiles) {
long useTime = System.currentTimeMillis() - startTime;
System.out.println("用时:" + useTime /1000f + "秒");
for(Map.Entry<String, List<String>> entry: repeatFiles.entrySet()){
System.out.println(entry.getKey());
for(String path: entry.getValue()){
System.out.println(path);
}
System.out.println();
}
}
});
rfm.start(new File("Z:\\娱乐"));
}
}
![](https://i-blog.csdnimg.cn/blog_migrate/e8f5903609226e01cdf5e33003e5b276.png)