在平时的项目维护中,我们经常会操作大批量的数据。
比如,我需要把某个excel中所有的手机号复制出来,然后去重。
对于这样的需求,通过今天分享的这个小工具类,可以轻松搞定。
mark一下吧!
package com.yangcq.learning.hantang.utils;
import lombok.extern.slf4j.Slf4j;
import java.util.*;
/**
* List去重
*/
@Slf4j
public class DuplicateRemovalUtil {
/**
* 文件路径
*/
private static final String SOURCE_FILE_PATH = "D://import/src_bill.txt";
private static final String TARGET_FILE_PATH = "D://import/bill.txt";
public static void main(String[] args) {
// 读取文件字符串列表,返回去重后的集合
List<String> srcList = ReadTextFileUtil.readTxt(SOURCE_FILE_PATH);
List<String> targetList = DuplicateRemovalUtil.listDuplicateRemoval(srcList);
log.info("源文件:{},源文件数据量:{}", SOURCE_FILE_PATH, srcList.size());
log.info("目标文件:{},目标文件数据量:{}", TARGET_FILE_PATH, targetList.size());
// 写去重后的数据到文件
WriteTextFileUtil.write2File(targetList, TARGET_FILE_PATH);
}
/**
* 使用LinkedHashSet去重,去重后顺序不变
*
* @param srcList
* @return 去重后的list
*/
private static List<String> listDuplicateRemoval(List<String> srcList) {
LinkedHashSet<String> hashSet = new LinkedHashSet<>(srcList);
List<String> targetList = new ArrayList<>(hashSet);
return targetList;
}
/**
* 通过set去重, 不打乱原有list的顺序
*
* @param list
* @return
*/
public List<String> distinctBySetOrder(List<String> list) {
Set<String> set = new HashSet<>();
List<String> newList = new ArrayList<>();
for (String t : list) {
if (set.add(t)) {
newList.add(t);
}
}
return newList;
}
}
这里有2个路径,解释一下:
/** * 源文件路径 */ private static final String SOURCE_FILE_PATH = "D://import/src_bill.txt"; /** * 目标文件路径 */ private static final String TARGET_FILE_PATH = "D://import/bill.txt";
读取文件工具类
package com.yangcq.learning.hantang.utils;
import lombok.extern.slf4j.Slf4j;
import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.List;
/**
* 读取文件工具类:去空格
*/
@Slf4j
public class ReadTextFileUtil {
public static List<String> readTxt(String filePath) {
InputStream is = null;
List<String> list = new ArrayList<>();
try {
is = new FileInputStream(filePath);
BufferedReader br = new BufferedReader(new InputStreamReader(is, "GBK"));
String str;
while ((str = br.readLine()) != null) {
list.add(str.trim());
}
} catch (Exception e) {
log.error("读取文件 error ", e);
} finally {
try {
is.close();
} catch (Exception e) {
log.error("关闭流 error ", e);
}
}
return list;
}
}
写文件工具类
package com.yangcq.learning.hantang.utils;
import lombok.extern.slf4j.Slf4j;
import java.io.FileWriter;
import java.io.IOException;
import java.util.List;
@Slf4j
public class WriteTextFileUtil {
public static void write2File(List<String> list, String filePath) {
FileWriter fileWriter = null;
try {
fileWriter = new FileWriter(filePath);
for (String str : list) {
fileWriter.write(str + "\r\n");
}
fileWriter.flush();
fileWriter.close();
} catch (IOException e) {
log.error("write2File error ", e);
}
}
}