前言
最近突然冒出了一个想法,利用RandomAccessFile和多线程实现文件的分段拷贝,跟直接用流复制,会不会耗时更短呢? 本来想着,多线程同时拷贝,耗时应该会更短。
但事实上,并非如此。绝大多数情况会更慢,不管线程开多少个。原因可能如下:
1、RandomAccessFile 效率更能没有 Buffer 流高
2、线程开的越多,线程调度切换的代价也会变高
3、由于多线程分段拷贝,底层磁盘的磁头频繁变化,增加了磁盘寻道时间和定位时间。换言之,对于多线程拷贝以及直接通过流复制,都是将磁盘数据先读到内存,再写入到磁盘副本。但是多线程拷贝会更慢的主要原因是:底层磁盘IO的速度的限制了CPU的速度【最主要!!!】
所以 利用RandomAccessFile实现多线程分段拷贝,是没有什么实际意义的。但是代码还是记录一下,因为,逻辑设计思维还是有借鉴意义的。代码实现过程中,还涉及到多线程并发的问题。
PS: 操作系统我学得是真滴菜,但老师讲得也水。。。
代码
package net.ysq.nio.test;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadPoolExecutor;
/**
* 复制者。负责调度多线程复制src到dest
*
* @author passerbyYSQ
* @date 2020-11-5 16:11:40
*/
public class MultiThreadCopyer {
private File src;
private File dest;
private String newName;
private long totalSize;
// 各个线程已经复制的大小
private long[] copied;
private int lastProgress;
// 专门负责从src到dest复制工作的线程池
// 并不是为每一个MultiThreadCopyer实例对象创建一个线程池
// 而是所有MultiThreadCopyer实例对象都是用这一线程池,故声明为静态变量
private static ThreadPoolExecutor threadPool;
// 线程数量
private int threadCount;
private CountDownLatch countDownLatch;
private CopyingListener listener;
static {
threadPool = (ThreadPoolExecutor) Executors.newFixedThreadPool(32);
}
public MultiThreadCopyer(File src, File dest, CopyingListener listener) {
this(src, dest, null, 14, listener);
}
public MultiThreadCopyer(File src, File dest, String newName, int threadCount,
CopyingListener listener) {
if (src.isDirectory()) {
throw new RuntimeException("不支持复制目录");
}
if (!dest.exists()) {
boolean isCreated = dest.mkdirs();
if (!isCreated) {
throw new RuntimeException("创建目标目录失败");
}
}
this.newName = (newName == null || "".equals(newName)) ? src.getName() : newName;
File tempDest = new File(dest, this.newName);
if (tempDest.exists()) {
// boolean isDel = tempDest.delete(); // 目标目录存在同名文件,则先删除
// if (!isDel) {
// throw new RuntimeException("删除目标目录同名文件失败");
// }
}
this.src = src;
this.dest = tempDest;
this.totalSize = this.src.length();
this.threadCount = (threadCount <= 0) ? 4 : threadCount;
this.countDownLatch = new CountDownLatch(this.threadCount);
this.copied = new long[ this.threadCount ];
this.listener = listener;
}
public void copy() {
long startTime = System.currentTimeMillis();
long sectionSize = (long) (totalSize / threadCount); // 取下整
// System.out.println("totalSize=" + totalSize);
// System.out.println("sectionSize=" + sectionSize);
long start = 0, end = sectionSize;
for(int i = 0; i < threadCount; i++) {
CopyWorker worker = new CopyWorker(start, end, i);
threadPool.execute(worker); // 交由线程池调度
// System.out.println(i + ":start=" + start + "; end=" + end);
start = end;
end = (i < threadCount - 1) ? (start + sectionSize) : totalSize;
}
try {
// 阻塞等待, 直至所有线程执行完成。CountDownLatch里面有线程同步
countDownLatch.await();
long endTime = System.currentTimeMillis();
if (listener != null) {
listener.onCompleted(dest, endTime - startTime);
}
} catch (InterruptedException e) {
e.printStackTrace();
}
}
// 计算进度
private void calculateProgress() {
if (listener != null) {
long totalCopied = 0;
for(int i = 0; i < threadCount; i++) {
totalCopied += copied[i];
}
int progress = (int) (100.0 * totalCopied / totalSize);
if (progress > lastProgress) { // 排队前先过滤。增加效率
synchronized (this) { // 防止多个线程同时,刷新同一个百分点,造成百分点错乱
if (progress > lastProgress) { // 抢到锁之后,再次验证
listener.onProgress(progress); // 回调进度
lastProgress = progress; // 记录上一个百分点
}
}
}
}
}
// 复制的线程
class CopyWorker extends Thread {
private RandomAccessFile randSrc;
private RandomAccessFile randDest;
// 负责的区间[start, end)
private long start;
private long end;
private int num; // 线程编号
private String threadName;
public CopyWorker(long start, long end, int num) {
super(src.getName() + "=>" + newName + ": thread-" + num);
this.start = start;
this.end = end;
this.num = num;
this.threadName = "thread-" + num;
}
@Override
public void run() {
try {
randSrc = new RandomAccessFile(src, "r");
randDest = new RandomAccessFile(dest, "rw");
// long startTime = System.currentTimeMillis();
randSrc.seek(start);
randDest.seek(start);
// long endTime = System.currentTimeMillis();
// System.out.println(num + " seek耗时:" + (endTime-startTime));
byte[] buf = new byte[1024]; // 缓冲的字节数组
long sum = 0; // 该区间内已复制的长度
int remain;
while (true) {
remain = (int) Math.min((end - start) - sum, buf.length);
int read = randSrc.read(buf, 0, remain);
randDest.write(buf, 0, read);
sum += read;
copied[num] = sum;
// 计算并回调进度
calculateProgress();
if (sum >= (end - start)) {
// 该区间复制完成
System.out.println(threadName + " 已完成区间复制");
countDownLatch.countDown();
break; // 不要忘了
}
}
} catch (IOException e) {
e.printStackTrace();
}
}
}
// 监听器
interface CopyingListener {
void onProgress(int progress); // 回调复制进度,百分数
void onCompleted(File dest, long millisecond); // 下载完成,回调 目标文件和耗时(毫秒)
}
}
测试
package net.ysq.nio.test;
import java.io.File;
import java.io.IOException;
import org.junit.Test;
/**
* @author passerbyYSQ
* @date 2020-11-5 18:05:49
*/
public class MultiThreadUtil {
File src = new File("E:\\Download\\cn_sql_server_2016_enterprise_x64_dvd_8699450.iso");
File dest = new File("E:\\");
@Test
public void test() {
new MultiThreadCopyer(src, dest, new MultiThreadCopyer.CopyingListener() {
@Override
public void onProgress(int progress) {
System.out.println("下载进度:" + progress + "%");
}
@Override
public void onCompleted(File dest, long millisecond) {
System.out.println("下载完成:");
System.out.println("目标文件:" + dest.getAbsolutePath());
System.out.println("耗时:" + millisecond + " ms");
}
}).copy(); // 不要忘了 .copy()
}
@Test
public void test2() throws IOException {
long startTime = System.currentTimeMillis();
dest = new File("E:\\abc.iso");
StreamUtil.copy(src, dest);
long endTime = System.currentTimeMillis();
System.out.println("耗时:" + (endTime - startTime));
}
}