去除重复的字符串java工具

/**
 * RepeatServiceTest.java
 * com.eetop.spider.util.service
 *
 * Function： TODO 
 *
 *   ver     date      		author
 * ──────────────────────────────────
 *   		 2012-8-23 		zhangzenglun
 *
 * Copyright (c) 2012, TNT All Rights Reserved.
*/

package com.eetop.spider.util.service;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.util.HashSet;
import java.util.Iterator;

import org.junit.Test;

/**
 * ClassName:RepeatServiceTest
 *
 * @author   zhangzenglun
 * @version  1.0
 * @since    v1.0
 * @Date	 2012-8-23 上午11:24:51 
 */
public class RepeatServiceTest {


	@Test
	public void testRepeat(){
		filterFileRepeatContent("J:\\xx\\云录\\insert_job.sql","J:\\xx\\云录\\insert_job_new.sql","GBK");
	}

	private void filterFileRepeatContent(String oriFile,String destFile,String charSet) {
		HashSet<String> set = new HashSet<String>();
		FileInputStream fin = null;
		FileOutputStream fout = null;
		BufferedReader br = null;
		BufferedWriter bw = null;
		try {
			fin = new FileInputStream(oriFile);
			fout = new FileOutputStream(destFile);
			br = new BufferedReader(new InputStreamReader(fin,charSet));
			bw = new BufferedWriter(new OutputStreamWriter(fout,charSet));
			String line = null;
			while((line = br.readLine())!=null) {
				set.add(line);
			}

			Iterator<String> it = set.iterator();
			String sql = null;
			while(it.hasNext()) {
				sql = it.next();
				System.out.println(sql);
				bw.write(sql);
				bw.newLine();
			}

		} catch (Exception e) {
			e.printStackTrace();
		} finally {
			if (br!=null) {
				try {
					br.close();
				} catch (IOException e) {
					e.printStackTrace();
				}
			}
			if (fin != null) {
				try {
					fin.close();
				} catch (IOException e) {
					e.printStackTrace();
				}
			}
			if (bw!=null) {
				try {
					bw.close();
				} catch (IOException e) {
					e.printStackTrace();
				}
			}
			if (fout != null) {
				try {
					fout.close();
				} catch (IOException e) {
					e.printStackTrace();
				}
			}
		}
	}
}