单线程写 多线程读 java6_java多线程合并文件-生产者模式-多线程读单线程写

上次那个工作线程又读又写的感觉在写上面竞争太激烈了，于是试了试一个线程来写。

阻塞队列维护读取缓存，给定长度，虽然本来就假设忽略了内存问题，但至少要我能跑起来嘛。

对此做了个简单的不科学的不严谨的不靠谱的比较：

文件数(112k/个)

程序一：同读写(毫秒)

程序二：多读一写

程序一：每个文件用时

程序二：每个文件用时

10

267

179

26.7

17.9

50

684

387

13.68

7.74

100

1285

437

12.85

4.37

200

1829

1236

9.145

6.18

500

7162

1762

14.324

3.524

1000

12992

2284

12.992

2.284

一言不合贴代码：

import java.io.BufferedReader;

import java.io.File;

import java.io.FileInputStream;

import java.io.FileNotFoundException;

import java.io.FileWriter;

import java.io.FilenameFilter;

import java.io.IOException;

import java.io.InputStreamReader;

import java.util.Date;

import java.util.concurrent.ArrayBlockingQueue;

import java.util.concurrent.CountDownLatch;

import java.util.concurrent.ExecutorService;

import java.util.concurrent.Executors;

import java.util.logging.FileHandler;

import java.util.logging.Level;

import java.util.logging.Logger;

import java.util.logging.SimpleFormatter;

public class TxtMergePC extends Thread {

/**

* 日志

*/

private static Logger logger;

/**

* 日志路径

*/

private static String logPath = "D:\\text\\log\\logPC.log";

/**

* 日志配置

*/

static {

logger = Logger.getLogger("txtMerge");

FileHandler fh;

try {

fh = new FileHandler(logPath, true);

logger.addHandler(fh);// 日志输出文件

logger.setLevel(Level.ALL);

fh.setFormatter(new SimpleFormatter());// 输出格式

} catch (SecurityException e) {

logger.log(Level.SEVERE, "安全性错误", e);

} catch (IOException e) {

logger.log(Level.SEVERE, "读取文件日志错误", e);

}

}

private static void infoLog(String msg) {

logger.log(Level.INFO, msg);

}

private static void errorLog(String msg, Throwable e) {

logger.log(Level.SEVERE, msg, e);

}

/**

* 待合并文件夹路径

*/

private String path;

/**

* 输出路径(绝对路径)

*/

private String outPath;

/**

* 文件列表

*/

private File[] files;

/**

* 剩余未合并文件数

*/

private Integer fileNum;// 未处理的文件数

private Integer dealFileNum = 0;// 处理文件数

/**

* 工作线程总数

*/

private Integer threadNum;// 读线程数

private Integer queueSize;// 队列容量

private ArrayBlockingQueue queue;

public TxtMergePC(String path, String outPath, int threadNum, int queueSize) {

this.path = path;

this.outPath = outPath;

this.threadNum = threadNum;

this.queueSize = queueSize;

this.queue = new ArrayBlockingQueue(this.queueSize);

this.files = new File(this.path).listFiles(new FilenameFilter() {

@Override

public boolean accept(File dir, String name) {

if (name.endsWith(".txt"))

return true;

return false;

}

});

this.fileNum = this.files.length;

StringBuilder sBuilder = new StringBuilder(this.fileNum + "个文本文件， at:" + this.path);

for (File f : this.files) {

sBuilder.append("\n");

sBuilder.append(f.getName());

}

infoLog(sBuilder.toString());

}

@Override

public void run() {

if (this.fileNum <= 0) {

infoLog("无文件：" + this.path);

return;

}

CountDownLatch doneSignal = new CountDownLatch(this.threadNum + 1);// 结束信号，用于关闭写操作

CountDownLatch startSignal = new CountDownLatch(1);// 开始执行信号

ExecutorService threadPool = Executors.newCachedThreadPool();

final FileWriter fileWriter;

try {

fileWriter = new FileWriter(outPath);

} catch (IOException e) {

errorLog("创建输出文件失败", e);

return;

}

for (int i = 0; i < threadNum; i++) {

threadPool.execute(new Worker(startSignal, doneSignal));

}

startSignal.countDown();// 开始

long begin = new Date().getTime();

threadPool.execute(new Thread(new Runnable() {

@Override

public void run() {

infoLog("写线程 启动");

int i = 1;

while (dealFileNum < files.length) {

StringBuilder sb = null;

try {

sb = queue.take();

infoLog("写线程 - " + (i++));

} catch (InterruptedException e) {

errorLog("取队列缓存失败", e);

}

if (sb == null)

continue;

try {

fileWriter.write(sb.toString());

} catch (IOException e) {

errorLog("写失败", e);

}

}

doneSignal.countDown();

infoLog("写线程 结束");

}

}));

try {

doneSignal.await();// 等待其他线程结束

} catch (InterruptedException e) {

e.printStackTrace();

} finally {

try {

fileWriter.close();

} catch (IOException e) {

e.printStackTrace();

}

}

threadPool.shutdown();

long end = new Date().getTime();

infoLog("结束，用时：" + (end - begin) + " ms");

}

/**

* 获取一个未处理的文件

*

* @author LiuJie

* @time 2016年8月24日 下午10:43:15

* @return 处理完了返回null

*/

private File getFile() {

if (0 == fileNum) {

return null;

}

File file = null;

synchronized (this.fileNum) {

if (0 < fileNum) {

file = this.files[this.fileNum - 1];

this.fileNum--;

infoLog(Thread.currentThread().getName() + " 操作" + file.getName() + "，剩余：" + fileNum);

}

}

return file;

}

/**

* 工作线程，容器中拿不出文件时结束

*

* @author 刘杰

* @time 2016年8月24日 下午11:04:36

*/

private class Worker implements Runnable {

private CountDownLatch startSignal;

private CountDownLatch doneSignal;

public Worker(CountDownLatch startSignal, CountDownLatch doneSignal) {

this.startSignal = startSignal;

this.doneSignal = doneSignal;

}

@Override

public void run() {

infoLog(Thread.currentThread().getName() + " 就绪");

try {

startSignal.await();// 等待开始执行信号的发布

} catch (InterruptedException e1) {

errorLog("等待开始信号异常", e1);

}

infoLog(Thread.currentThread().getName() + " 开始");

FileInputStream fis = null;

InputStreamReader isr = null;

BufferedReader br = null;

while (true) {

File file = getFile();

if (null == file) {

break;

}

try {

fis = new FileInputStream(file);

isr = new InputStreamReader(fis, "UTF-8");

br = new BufferedReader(isr);

String readStr = "";

StringBuilder sb = new StringBuilder("\r\n" + file.getName() + "\r\n");

while ((readStr = br.readLine()) != null) {

sb.append(readStr);

}

try {

queue.put(sb);

} catch (InterruptedException e) {

errorLog("加入缓存队列失败-" + file.getName(), e);

}

} catch (FileNotFoundException e) {

errorLog("找不到指定文件", e);

} catch (IOException e) {

errorLog("读取文件失败", e);

} finally {

try {

br.close();

isr.close();

fis.close();

} catch (IOException e) {

errorLog("关闭异常", e);

}

dealFileNum++;

}

}

doneSignal.countDown();// 告诉“主线程”工作完成

infoLog(Thread.currentThread().getName() + " 结束");

}

}

public static void main(String[] args) {

TxtMergePC txtMerge = new TxtMergePC("D:\\text", "D:\\text\\merge\\mergePC.txt", 5, 5);

txtMerge.start();

}

}