详情说明
IO流操作只要使用 BufferedInputStream和BufferedOutputStream,之所以使用这两个,可以点击该网址了解
文件合并时使用FileChannel做文件合并
引入相关依赖
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.7</version>
</dependency>
实现
SliceInfo.class
@Data
@AllArgsConstructor
public class SliceInfo {
private long start;
private long end;
private long page;
}
@Slf4j
@Configuration
public class FileDownloadUtil {
private static Boolean enableSlice;
/**
* 临时目录
*/
private static final String tempDir = "D:/temp/";
/**
* 保存目录
*/
private static final String saveDir = "D:/target/";
/**
* 分片大小
*/
private static final Long BUFFER_SIZE = (long) (1024 * 1024 * 2);
@Value("${download.slice.enable}")
public void setFileUtil(Boolean slice) {
enableSlice = slice;
}
static {
File file = new File(tempDir);
if (!file.exists()) {
file.mkdir();
}
file = new File(saveDir);
if (!file.exists()) {
file.mkdir();
}
}
/**
* 下载文件
*
* @param fileUrl 下载路径
*/
public static String downloadFile(String fileUrl) {
String fileName = fileUrl.substring(fileUrl.lastIndexOf("/") + 1);
String filePath = saveDir + fileName;
if (new File(filePath).exists()) {
log.info("{}文件已存在,跳过", filePath);
return filePath;
}
try {
long start = System.currentTimeMillis();
if (!enableSlice) {
log.info("不使用分片下载");
fileDownload(fileUrl, null, null, filePath);
log.info("不使用分片下载耗时:{}", System.currentTimeMillis() - start);
return filePath;
}
log.info("使用分片下载");
downloadBySlice(fileUrl, fileName, filePath);
log.info("使用分片下载耗时:{}", System.currentTimeMillis() - start);
return filePath;
} catch (Exception e) {
log.error("文件下载异常:{}", ExceptionUtils.getStackTrace(e));
throw new RuntimeException(e.getMessage());
}
}
/**
* 下载片
* 分片下载主体方法
*
* @param fileUrl 文件下载路径
* @param fileName 文件名称
* @param filePath 文件路径
* @throws IOException ioexception
*/
private static void downloadBySlice(String fileUrl, String fileName, String filePath) throws IOException {
Long fileSize = isSupportSliceDownload(fileUrl);
if (Objects.isNull(fileSize)) {
throw new RuntimeException("不支持分片下载");
}
CopyOnWriteArrayList<SliceInfo> sliceInfoList = splitFileSize(fileSize);
CountDownLatch countDownLatch = new CountDownLatch(Math.toIntExact(sliceInfoList.size()));
ThreadPoolExecutor threadPoolExecutor = new ThreadPoolExecutor(4, 4, 30, TimeUnit.SECONDS, new LinkedBlockingQueue<>());
try {
//使用多线程进行分片下载
sliceInfoList.forEach(ele -> {
threadPoolExecutor.submit(() -> {
//临时文件目录
String tempPath = tempDir + ele.getPage() + "-" + fileName;
fileDownload(fileUrl, ele.getStart(), ele.getEnd(), tempPath);
countDownLatch.countDown();
log.info("文件下载进度:{}/{}", sliceInfoList.size() - countDownLatch.getCount(), sliceInfoList.size());
});
});
countDownLatch.await();
//合并文件
mergeFile(fileName, sliceInfoList.size(),tempDir, filePath);
} catch (InterruptedException e) {
log.error("文件合并异常,异常信息:{}", ExceptionUtils.getStackTrace(e));
throw new RuntimeException(e.getMessage());
} finally {
//每次执行完,必须关闭线程
threadPoolExecutor.shutdown();
}
}
/**
* 合并分片文件
*
* @param fileName 文件名称
* @param sliceLen 分片数量
* @param tempDir 临时文件的目录
* @param filePath 保存文件路径
*/
private static void mergeFile(String fileName, Integer sliceLen, String tempDir, String filePath) {
File targetFile = new File(filePath);
try (FileChannel targetChannel = new FileOutputStream(targetFile).getChannel()) {
for (int i = 0; i < sliceLen; i++) {
File file = new File(tempDir, i + "-" + fileName);
FileChannel tempChannel = new FileInputStream(file).getChannel();
long size = tempChannel.size();
for (long left = size; left > 0; ) {
left -= tempChannel.transferTo((size - left), left, targetChannel);
}
tempChannel.close();
file.delete();
}
} catch (IOException e) {
throw new RuntimeException(e);
}
}
private static CopyOnWriteArrayList<SliceInfo> splitFileSize(long fileSize) {
CopyOnWriteArrayList<SliceInfo> resultList = new CopyOnWriteArrayList<>();
long size = fileSize;
long left = 0;
long page = 0;
while (size > 0) {
long start = 0;
long end;
start = left;
//分页
if (size < BUFFER_SIZE) {
end = left + size;
} else {
end = left += BUFFER_SIZE;
}
size -= BUFFER_SIZE;
if (start != 0) {
start++;
}
final SliceInfo sliceInfo = new SliceInfo(start, end, page);
resultList.add(sliceInfo);
page++;
}
return resultList;
}
/**
* 是否支持分片下载,支持则返回文件长度
*
* @param uri 下载链接
* @return {@link Long} 文件大小
* @throws IOException ioexception
*/
public static Long isSupportSliceDownload(String uri) throws IOException {
URL url = new URL(uri);
HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection();
urlConnection.setRequestMethod("GET");
urlConnection.setRequestProperty("Range", "bytes=0-1");
urlConnection.connect();
String supportSlice = urlConnection.getHeaderField("Content-Range");
if (StringUtils.isBlank(supportSlice)) {
return null;
}
return Long.parseLong(supportSlice.split("/")[1]);
}
/**
* 文件下载
*
* @param fileUrl 下载路径url
* @param start 开始位置,为空,则非分片下载
* @param end 结束位置
* @param filePath 保存的文件路径
*/
public static void fileDownload(String fileUrl, Long start, Long end, String filePath) {
File file = new File(filePath);
if (file.exists()) {
log.info("{}文件已存在,跳过下载", filePath);
return;
}
HttpURLConnection urlConnection = null;
try {
URL url = new URL(fileUrl);
urlConnection = (HttpURLConnection) url.openConnection();
urlConnection.setRequestMethod("GET");
if (Objects.nonNull(start)) {
urlConnection.setRequestProperty("Range", "bytes=" + start + "-" + end);
}
urlConnection.setRequestProperty("Connection", "Keep-Alive");
urlConnection.connect();
} catch (IOException e) {
log.error("文件下载,连接异常,连接信息:{},异常信息:{}", JSON.toJSONString(urlConnection), ExceptionUtils.getStackTrace(e));
throw new RuntimeException(e.getMessage());
}
// 下面为文件写入
byte[] byteBuffer = new byte[Math.toIntExact(BUFFER_SIZE)];
try (BufferedInputStream bis = new BufferedInputStream(urlConnection.getInputStream());
BufferedOutputStream bos = new BufferedOutputStream(Files.newOutputStream(file.toPath()))) {
int len;
while ((len = bis.read(byteBuffer)) != -1){
bos.write(byteBuffer, 0, len);
}
bis.close();
bos.close();
} catch (Exception e) {
log.error("文件写入异常,异常信息:{}", ExceptionUtils.getStackTrace(e));
throw new RuntimeException(e);
} finally {
urlConnection.disconnect();
}
}
}