java多线程合并文件-多线程读写

2019独角兽企业重金招聘Python工程师标准>>>

先贴代码： 

package test;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileWriter;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.logging.ConsoleHandler;
import java.util.logging.FileHandler;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.logging.SimpleFormatter;

public class TxtMerge extends Thread {

	/**
	 * 日志
	 */
	private static Logger logger;

	/**
	 * 日志路径
	 */
	private static String logPath = "D:\\text\\log.log";

	/**
	 * 日志配置
	 */
	static {
		logger = Logger.getLogger("txtMerge");
		FileHandler fh;
		try {
			fh = new FileHandler(logPath, true);
			logger.addHandler(fh);// 日志输出文件
			logger.setLevel(Level.ALL);
			fh.setFormatter(new SimpleFormatter());// 输出格式
		} catch (SecurityException e) {
			logger.log(Level.SEVERE, "安全性错误", e);
		} catch (IOException e) {
			logger.log(Level.SEVERE, "读取文件日志错误", e);
		}
	}

	private static void infoLog(String msg) {
		logger.log(Level.INFO, msg);
	}

	private static void errorLog(String msg, Throwable e) {
		logger.log(Level.SEVERE, msg, e);
	}

	/**
	 * 待合并文件夹路径
	 */
	private String path;

	/**
	 * 输出路径（绝对路径）
	 */
	private String outPath;

	/**
	 * 文件列表
	 */
	private File[] files;

	/**
	 * 剩余未合并文件数
	 */
	private Integer fileNum;// 未处理的文件数

	/**
	 * 工作线程总数
	 */
	private Integer threadNum;

	/**
	 * 对合并文件进行写操作的对象
	 */
	private FileWriter fileWriter;

	/**
	 * 
	 * @param path
	 *            待合并文件夹路径
	 * @param outPath
	 *            输出路径（绝对路径）
	 * @param threadNum
	 *            工作线程总数
	 */
	public TxtMerge(String path, String outPath, int threadNum) {
		this.path = path;
		this.outPath = outPath;
		this.threadNum = threadNum;
		this.files = new File(this.path).listFiles(new FilenameFilter() {

			@Override
			public boolean accept(File dir, String name) {
				if (name.endsWith(".txt"))
					return true;
				return false;
			}
		});
		this.fileNum = this.files.length;
		StringBuilder sBuilder = new StringBuilder(this.fileNum + "个文本文件， at:" + this.path);
		for (File f : this.files) {
			sBuilder.append("\n");
			sBuilder.append(f.getName());
		}
		infoLog(sBuilder.toString());
	}

	@Override
	public void run() {
		if (this.fileNum <= 0) {
			infoLog("无文件：" + this.path);
			return;
		}
		CountDownLatch doneSignal = new CountDownLatch(this.threadNum);// 结束信号，用于关闭写操作
		CountDownLatch startSignal = new CountDownLatch(1);// 开始执行信号
		ExecutorService threadPool = Executors.newCachedThreadPool();
		try {
			this.fileWriter = new FileWriter(outPath);
		} catch (IOException e) {
			errorLog("创建输出文件失败", e);
			return;
		}
		for (int i = 0; i < threadNum; i++) {
			threadPool.execute(new Worker(startSignal, doneSignal));
		}
		startSignal.countDown();// 开始
		try {
			doneSignal.await();// 等待其他线程结束
		} catch (InterruptedException e) {
			e.printStackTrace();
		} finally {
			try {
				this.fileWriter.close();
			} catch (IOException e) {
				e.printStackTrace();
			}
		}
		threadPool.shutdown();
		infoLog("结束");
	}

	/**
	 * 获取一个未处理的文件
	 * 
	 * @author LiuJie
	 * @time 2016年8月24日 下午10:43:15
	 * @return 处理完了返回null
	 */
	private File getFile() {
		if (0 == fileNum) {
			return null;
		}
		File file = null;
		synchronized (this.fileNum) {
			if (0 < fileNum) {
				file = this.files[this.fileNum - 1];
				this.fileNum--;
				infoLog(Thread.currentThread().getName() + " 操作" + file.getName() + "，剩余：" + fileNum);
			}
		}
		return file;
	}

	/**
	 * 工作线程，容器中拿不出文件时结束
	 * 
	 * @author Liujie
	 * @time 2016年8月24日 下午11:04:36
	 */
	private class Worker implements Runnable {

		private CountDownLatch startSignal;
		private CountDownLatch doneSignal;

		public Worker(CountDownLatch startSignal, CountDownLatch doneSignal) {
			this.startSignal = startSignal;
			this.doneSignal = doneSignal;
		}

		@Override
		public void run() {
			infoLog(Thread.currentThread().getName() + " 就绪");
			try {
				startSignal.await();// 等待开始执行信号的发布
			} catch (InterruptedException e1) {
				errorLog("等待开始信号异常", e1);
			}
			infoLog(Thread.currentThread().getName() + " 开始");
			FileInputStream fis = null;
			InputStreamReader isr = null;
			BufferedReader br = null;
			while (true) {
				File file = getFile();
				if (null == file) {
					break;
				}
				try {
					fis = new FileInputStream(file);
					isr = new InputStreamReader(fis, "UTF-8");
					br = new BufferedReader(isr);
					String readStr = "";
					while ((readStr = br.readLine()) != null) {
						synchronized (fileWriter) {
							fileWriter.write(readStr + "\r\n");
						}
					}
				} catch (FileNotFoundException e) {
					errorLog("找不到指定文件", e);
				} catch (IOException e) {
					errorLog("读取文件失败", e);
				} finally {
					try {
						br.close();
						isr.close();
						fis.close();
					} catch (IOException e) {
						errorLog("关闭异常", e);
					}
				}
			}
			doneSignal.countDown();// 告诉“主线程”工作完成
			infoLog(Thread.currentThread().getName() + " 结束");
		}
	}

	public static void main(String[] args) throws InterruptedException {
		TxtMerge txtMerge = new TxtMerge("D:\\text", "D:\\text\\merge.txt", 5);
		txtMerge.start();
	}
}

 

转载于:https://my.oschina.net/liujiest/blog/737464