java 大文件去重_java 两个csv文件数据去重

packagecom.tangxin.kafka.service;importorg.apache.commons.io.FileUtils;importorg.apache.commons.io.LineIterator;importorg.springframework.util.StringUtils;import java.io.*;importjava.math.BigDecimal;import java.util.*;/*** 两个csv文件数据去重*/

public classCSVDeduplication {private static final String CSV_PATH = "I:\\";public static Listids(String path) {

List result = new ArrayList<>();

File csv= new File(path); //CSV文件路径

LineIterator it = null;try{

it=FileUtils.lineIterator(csv);while(it.hasNext()) {

String line=it.nextLine();if (line.trim().contains("id")) {continue;

}

String[] arr= line.split(",");

String id= arr[0];

id= id.replaceAll("\"", "").trim();

result.add(id);

}

}catch(Exception e) {

}finally{