set 去重

转载 Java合并两个文件，Set去重。 收藏

package com.joyhoo.words;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Set;

public class HashSetInFile
{


//simpleFile、insertFile为两个输入文件，outFile为输出文件
public void readFile(String simpleFile,String insertFile,String outFile)
{
Set set = new HashSet();

try
{
String line="";

FileInputStream fisf = new FileInputStream(simpleFile);
InputStreamReader isrf = new InputStreamReader(fisf, "UTF8");
BufferedReader br =new BufferedReader(isrf);

while((line=br.readLine())!=null ){

if(!line.contains(" "))
set.add(line);
}
br.close();
System.out.println(set.size());

FileInputStream fis = new FileInputStream(insertFile);
InputStreamReader isr = new InputStreamReader(fis, "UTF8");
BufferedReader inbr=new BufferedReader(isr);

String temp = "";
while((temp = inbr.readLine()) != null){
if(!set.contains(temp) ){
set.add(temp);
}
}
inbr.close();

FileOutputStream fos = new FileOutputStream(outFile);
BufferedWriter out = new BufferedWriter(new OutputStreamWriter(fos,"UTF8"));
Iterator it = set.iterator();

while(it.hasNext()){

String element = (String)it.next();

out.write(element);
out.newLine();

}
out.close();

}catch(Exception e){
System.out.println("readFile errors:" + e.getMessage());
}

}

public static void main(String[] args)
{
HashSetInFile hf = new HashSetInFile();

String simple = "D:\\tmp\\simplexu8.txt";

String insert = "D:\\tmp\\sogou.txt";

String outfile = "D:\\tmp\\simplexu81.txt";

hf.readFile(simple, insert, outfile);
}

}