package com.jmyd.commons.plugins.manager; import com.jmyd.commons.plugins.task.DownloadTask; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; import java.io.RandomAccessFile; import java.net.HttpURLConnection; import java.net.URL; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; /** * @date 2022/2/25 16:36 */ public class DownLoadManager { private static final Logger LOGGER = LoggerFactory.getLogger(DownLoadManager.class); /** * 每个线程下载的字节数 */ private long unitSize = 1000 * 1024; /** * 创建线程池 */ private ExecutorService taskExecutor = Executors.newFixedThreadPool(10); private CloseableHttpClient httpClient; public DownLoadManager() { PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(); cm.setMaxTotal(100); httpClient = HttpClients.custom().setConnectionManager(cm).build(); } /** * 启动多个线程下载文件 * * @param remoteFileUrl 远程下载地址 * @param localPath 本地保存地址 * @author samphin * @date 2022-2-28 09:46:32 */ public void download(String remoteFileUrl, String localPath) throws IOException { String fileName = new URL(remoteFileUrl).getFile(); System.out.println("远程文件名称:" + fileName); fileName = fileName.substring(fileName.lastIndexOf("/") + 1).replace("%20", " "); System.out.println("本地文件名称:" + fileName); long fileSize = this.getRemoteFileSize(remoteFileUrl); this.createFile(localPath + fileName, fileSize); long threadCount = (fileSize / unitSize) + (fileSize % unitSize != 0 ? 1 : 0); long offset = 0; CountDownLatch end = new CountDownLatch(Long.valueOf(threadCount).intValue()); if (fileSize <= unitSize) {// 如果远程文件尺寸小于等于unitSize DownloadTask downloadThread = new DownloadTask(remoteFileUrl, localPath + fileName, offset, fileSize, end, httpClient); taskExecutor.execute(downloadThread); } else {// 如果远程文件尺寸大于unitSize for (int i = 1; i < threadCount; i++) { DownloadTask downloadThread = new DownloadTask(remoteFileUrl, localPath + fileName, offset, unitSize, end, httpClient); taskExecutor.execute(downloadThread); offset = offset + unitSize; } if (fileSize % unitSize != 0) {// 如果不能整除,则需要再创建一个线程下载剩余字节 DownloadTask downloadThread = new DownloadTask(remoteFileUrl, localPath + fileName, offset, fileSize - unitSize * (threadCount - 1), end, httpClient); taskExecutor.execute(downloadThread); } } try { end.await(); } catch (InterruptedException e) { LOGGER.error("DownLoadManager exception msg:{}", e.getMessage()); e.printStackTrace(); } taskExecutor.shutdown(); LOGGER.info("下载完成!{} ", localPath + fileName); } /** * 获取远程文件尺寸 */ private long getRemoteFileSize(String remoteFileUrl) throws IOException { long fileSize = 0; HttpURLConnection httpConnection = (HttpURLConnection) new URL(remoteFileUrl).openConnection(); //使用HEAD方法 httpConnection.setRequestMethod("HEAD"); int responseCode = httpConnection.getResponseCode(); if (responseCode >= 400) { LOGGER.error("Web服务器响应错误!"); return 0; } String sHeader; for (int i = 1; ; i++) { sHeader = httpConnection.getHeaderFieldKey(i); if (sHeader != null && sHeader.equals("Content-Length")) { System.out.println("文件大小ContentLength:" + httpConnection.getContentLength()); fileSize = Long.parseLong(httpConnection.getHeaderField(sHeader)); break; } } return fileSize; } /** * 创建指定大小的文件 */ private void createFile(String fileName, long fileSize) throws IOException { File newFile = new File(fileName); if (!newFile.exists()) { newFile.getParentFile().mkdirs(); newFile.createNewFile(); } RandomAccessFile raf = new RandomAccessFile(newFile, "rw"); raf.setLength(fileSize); raf.close(); } }
package com.jmyd.commons.plugins.task; import org.apache.http.client.ClientProtocolException; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.protocol.BasicHttpContext; import org.apache.http.protocol.HttpContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.BufferedInputStream; import java.io.File; import java.io.IOException; import java.io.RandomAccessFile; import java.util.concurrent.CountDownLatch; /** * @date 2022/2/25 16:36 */ public class DownloadTask implements Runnable { private static final Logger LOGGER = LoggerFactory.getLogger(DownloadTask.class); /** * 待下载的文件 */ private String url = null; /** * 本地文件名 */ private String fileName = null; /** * 偏移量 */ private long offset = 0; /** * 分配给本线程的下载字节数 */ private long length = 0; private CountDownLatch end; private CloseableHttpClient httpClient; private HttpContext context; /** * @param url 下载文件地址 * @param offset 本线程下载偏移量 * @param length 本线程下载长度 */ public DownloadTask(String url, String file, long offset, long length, CountDownLatch end, CloseableHttpClient httpClient) { this.url = url; this.fileName = file; this.offset = offset; this.length = length; this.end = end; this.httpClient = httpClient; this.context = new BasicHttpContext(); LOGGER.debug("偏移量=" + offset + ";字节数=" + length); } @Override public void run() { try { HttpGet httpGet = new HttpGet(this.url); httpGet.addHeader("Range", "bytes=" + this.offset + "-" + (this.offset + this.length - 1)); httpGet.addHeader("Referer", "http://api.bilibili.com"); CloseableHttpResponse response = httpClient.execute(httpGet, context); BufferedInputStream bis = new BufferedInputStream(response.getEntity().getContent()); byte[] buff = new byte[1024]; int bytesRead; File newFile = new File(fileName); RandomAccessFile raf = new RandomAccessFile(newFile, "rw"); while ((bytesRead = bis.read(buff, 0, buff.length)) != -1) { raf.seek(this.offset); raf.write(buff, 0, bytesRead); this.offset = this.offset + bytesRead; } raf.close(); bis.close(); } catch (ClientProtocolException e) { LOGGER.error("DownloadThread exception msg:{}", e.getMessage()); } catch (IOException e) { LOGGER.error("DownloadThread exception msg:{}", e.getMessage()); } finally { end.countDown(); LOGGER.info(end.getCount() + " is go on!"); System.out.println(end.getCount() + " is go on!"); } } }
/**
* 单元测试
* @date 2022/2/25 16:36
*/
@Test public void downloadTest() throws Exception { String remoteFileUrl = "远程文件下载地址"; //本地保存的目录地址 String localPath = "d://tmp/"; new DownLoadManager().download(url, localPath); }