上次那个工作线程又读又写的感觉在写上面竞争太激烈了，于是试了试一个线程来写。
阻塞队列维护读取缓存，给定长度，虽然本来就假设忽略了内存问题，但至少要我能跑起来嘛。
对此做了个简单的不科学的不严谨的不靠谱的比较：
| 文件数（112k/个） | 程序一：同读写（毫秒） | 程序二：多读一写 | 程序一：每个文件用时 | 程序二：每个文件用时 |
| 10 | 267 | 179 | 26.7 | 17.9 |
| 50 | 684 | 387 | 13.68 | 7.74 |
| 100 | 1285 | 437 | 12.85 | 4.37 |
| 200 | 1829 | 1236 | 9.145 | 6.18 |
| 500 | 7162 | 1762 | 14.324 | 3.524 |
| 1000 | 12992 | 2284 | 12.992 | 2.284 |
一言不合贴代码：
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileWriter;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.Date;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.logging.FileHandler;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.logging.SimpleFormatter;
public class TxtMergePC extends Thread {
/**
* 日志
*/
private static Logger logger;
/**
* 日志路径
*/
private static String logPath = "D:\\text\\log\\logPC.log";
/**
* 日志配置
*/
static {
logger = Logger.getLogger("txtMerge");
FileHandler fh;
try {
fh = new FileHandler(logPath, true);
logger.addHandler(fh);// 日志输出文件
logger.setLevel(Level.ALL);
fh.setFormatter(new SimpleFormatter());// 输出格式
} catch (SecurityException e) {
logger.log(Level.SEVERE, "安全性错误", e);
} catch (IOException e) {
logger.log(Level.SEVERE, "读取文件日志错误", e);
}
}
private static void infoLog(String msg) {
logger.log(Level.INFO, msg);
}
private static void errorLog(String msg, Throwable e) {
logger.log(Level.SEVERE, msg, e);
}
/**
* 待合并文件夹路径
*/
private String path;
/**
* 输出路径（绝对路径）
*/
private String outPath;
/**
* 文件列表
*/
private File[] files;
/**
* 剩余未合并文件数
*/
private Integer fileNum;// 未处理的文件数
private Integer dealFileNum = 0;// 处理文件数
/**
* 工作线程总数
*/
private Integer threadNum;// 读线程数
private Integer queueSize;// 队列容量
private ArrayBlockingQueue<StringBuilder> queue;
public TxtMergePC(String path, String outPath, int threadNum, int queueSize) {
this.path = path;
this.outPath = outPath;
this.threadNum = threadNum;
this.queueSize = queueSize;
this.queue = new ArrayBlockingQueue<StringBuilder>(this.queueSize);
this.files = new File(this.path).listFiles(new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
if (name.endsWith(".txt"))
return true;
return false;
}
});
this.fileNum = this.files.length;
StringBuilder sBuilder = new StringBuilder(this.fileNum + "个文本文件， at:" + this.path);
for (File f : this.files) {
sBuilder.append("\n");
sBuilder.append(f.getName());
}
infoLog(sBuilder.toString());
}
@Override
public void run() {
if (this.fileNum <= 0) {
infoLog("无文件：" + this.path);
return;
}
CountDownLatch doneSignal = new CountDownLatch(this.threadNum + 1);// 结束信号，用于关闭写操作
CountDownLatch startSignal = new CountDownLatch(1);// 开始执行信号
ExecutorService threadPool = Executors.newCachedThreadPool();
final FileWriter fileWriter;
try {
fileWriter = new FileWriter(outPath);
} catch (IOException e) {
errorLog("创建输出文件失败", e);
return;
}
for (int i = 0; i < threadNum; i++) {
threadPool.execute(new Worker(startSignal, doneSignal));
}
startSignal.countDown();// 开始
long begin = new Date().getTime();
threadPool.execute(new Thread(new Runnable() {
@Override
public void run() {
infoLog("写线程 启动");
int i = 1;
while (dealFileNum < files.length) {
StringBuilder sb = null;
try {
sb = queue.take();
infoLog("写线程 - " + (i++));
} catch (InterruptedException e) {
errorLog("取队列缓存失败", e);
}
if (sb == null)
continue;
try {
fileWriter.write(sb.toString());
} catch (IOException e) {
errorLog("写失败", e);
}
}
doneSignal.countDown();
infoLog("写线程 结束");
}
}));
try {
doneSignal.await();// 等待其他线程结束
} catch (InterruptedException e) {
e.printStackTrace();
} finally {
try {
fileWriter.close();
} catch (IOException e) {
e.printStackTrace();
}
}
threadPool.shutdown();
long end = new Date().getTime();
infoLog("结束，用时：" + (end - begin) + " ms");
}
/**
* 获取一个未处理的文件
*
* @author LiuJie
* @time 2016年8月24日 下午10:43:15
* @return 处理完了返回null
*/
private File getFile() {
if (0 == fileNum) {
return null;
}
File file = null;
synchronized (this.fileNum) {
if (0 < fileNum) {
file = this.files[this.fileNum - 1];
this.fileNum--;
infoLog(Thread.currentThread().getName() + " 操作" + file.getName() + "，剩余：" + fileNum);
}
}
return file;
}
/**
* 工作线程，容器中拿不出文件时结束
*
* @author 刘杰
* @time 2016年8月24日 下午11:04:36
*/
private class Worker implements Runnable {
private CountDownLatch startSignal;
private CountDownLatch doneSignal;
public Worker(CountDownLatch startSignal, CountDownLatch doneSignal) {
this.startSignal = startSignal;
this.doneSignal = doneSignal;
}
@Override
public void run() {
infoLog(Thread.currentThread().getName() + " 就绪");
try {
startSignal.await();// 等待开始执行信号的发布
} catch (InterruptedException e1) {
errorLog("等待开始信号异常", e1);
}
infoLog(Thread.currentThread().getName() + " 开始");
FileInputStream fis = null;
InputStreamReader isr = null;
BufferedReader br = null;
while (true) {
File file = getFile();
if (null == file) {
break;
}
try {
fis = new FileInputStream(file);
isr = new InputStreamReader(fis, "UTF-8");
br = new BufferedReader(isr);
String readStr = "";
StringBuilder sb = new StringBuilder("\r\n" + file.getName() + "\r\n");
while ((readStr = br.readLine()) != null) {
sb.append(readStr);
}
try {
queue.put(sb);
} catch (InterruptedException e) {
errorLog("加入缓存队列失败-" + file.getName(), e);
}
} catch (FileNotFoundException e) {
errorLog("找不到指定文件", e);
} catch (IOException e) {
errorLog("读取文件失败", e);
} finally {
try {
br.close();
isr.close();
fis.close();
} catch (IOException e) {
errorLog("关闭异常", e);
}
dealFileNum++;
}
}
doneSignal.countDown();// 告诉“主线程”工作完成
infoLog(Thread.currentThread().getName() + " 结束");
}
}
public static void main(String[] args) {
TxtMergePC txtMerge = new TxtMergePC("D:\\text", "D:\\text\\merge\\mergePC.txt", 5, 5);
txtMerge.start();
}
}
2969
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
