记录一些学习分片下载的细节~
首先准备一些工具类，文件内容获取，和目标源连接的一些方法
public class FileUtils { public static long getFileSize(String url) { File file = new File(url); return file.exists() && file.isFile() ? file.length() : 0; } }/** * @author Zhongzy * @description * @since 2023-10-19 9:42 */ public class HttpUtils { /** * @param url 目标源 * @param startPoint 区域块起始位置 * @param endPoint 区域快结束位置 * @return */ public static HttpURLConnection httpURLConnection(String url, long startPoint, long endPoint) { //链接目标源 HttpURLConnection httpURLConnection = httpURLConnection(url); System.out.println(Thread.currentThread().getName()+"下载区间是"+startPoint+"-"+endPoint); if (0 != endPoint) {//这里是向服务器申请分片下载的关键，将目标资源的数据区间传过去，可以下载对应的区间 httpURLConnection.setRequestProperty("RANGE", "bytes=" + startPoint + "-" + endPoint); } else {//如果为最后一块时不传入end就是将剩余的全部下载 httpURLConnection.setRequestProperty("RANGE", "bytes=" + startPoint + "-"); } return httpURLConnection; } /** * 下载 * * @param url 获取下载地址 * @return */ public static HttpURLConnection httpURLConnection(String url) { try { //创建目标源链接 URL httpUrl = new URL(url); HttpURLConnection httpURLConnection = (HttpURLConnection) httpUrl.openConnection(); //向服务器发送标识信息,这里的标识信息可以自行百度 httpURLConnection.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.106 Safari/537.36"); return httpURLConnection; } catch (IOException e) { throw new RuntimeException(e); } } /** * 文件名获取 * * @param url 下载路径 * @return 文件名 */ public static String getFileName(String url) { int index = url.lastIndexOf("/"); return url.substring(index + 1); } /** * 返回下载长度 * * @param url * @return */ public static long getFileLength(String url) { int length; HttpURLConnection httpURLConnection; httpURLConnection = httpURLConnection(url); length = httpURLConnection.getContentLength(); //关闭连接 httpURLConnection.disconnect(); return length; } }
写一个下载需要的参数的实体类
public class DownLoadTask implements Callable<Boolean> {//用callable是为了拿一个返回值 private String url;//下载地址 private long startPoint;//初始下载地址 private long endPoint;//结束下载地址 private int index;//当前标识位置 private DownLoadInfoThread downLoadInfoThread;//处理当前下载信息 private CountDownLatch countDownLatch;//线程计数器 public DownLoadTask(String url, long startPoint, long endPoint, int index, DownLoadInfoThread downLoadInfoThread, CountDownLatch countDownLatch) { this.url = url; this.startPoint = startPoint; this.endPoint = endPoint; this.index = index; this.downLoadInfoThread = downLoadInfoThread; this.countDownLatch = countDownLatch; } }完整代码及call实现
public class DownLoadTask implements Callable<Boolean> { private String url;//下载地址 private long startPoint;//初始下载地址 private long endPoint;//结束下载地址 private int index;//当前标识位置 private DownLoadInfoThread downLoadInfoThread;//处理当前下载信息 private CountDownLatch countDownLatch;//线程计数器 public DownLoadTask(String url, long startPoint, long endPoint, int index, DownLoadInfoThread downLoadInfoThread, CountDownLatch countDownLatch) { this.url = url; this.startPoint = startPoint; this.endPoint = endPoint; this.index = index; this.downLoadInfoThread = downLoadInfoThread; this.countDownLatch = countDownLatch; } @Override public Boolean call() throws Exception { //下载地址 String fileName = HttpUtils.getFileName(url); //拼接下载路径 fileName = fileName + ".temp" + index; fileName = PathEnum.Down_Path.getPath() + fileName; //创建目标源链接 HttpURLConnection httpURLConnection = HttpUtils.httpURLConnection(url, startPoint, endPoint); //文件流操作 try ( //获取输入流 InputStream is = httpURLConnection.getInputStream(); //包装为缓冲流 BufferedInputStream bis = new BufferedInputStream(is); //断点下载需要用到随机文件,操作权限read-write RandomAccessFile raf = new RandomAccessFile(fileName, "rw"); ) { //文件流操作 int length = -1; byte[] buffer = new byte[1024 * 1000]; while ((length = bis.read(buffer)) != -1) { downLoadInfoThread.downSize.add(length); raf.write(buffer, 0, length); } System.out.println("thread" + Thread.currentThread().getName() + "正在运行"); } catch (FileNotFoundException e) { System.err.println("-error-文件没有找到"); return false; } catch (IOException e) { System.err.println("-error-下载错误"); return false; } finally { httpURLConnection.disconnect(); countDownLatch.countDown(); } return true; } }
单独开一个线程记录下载信息，模拟下载中的速度，进度
public class DownLoadInfoThread implements Runnable { private static final double MB = 1024d * 1024d; private long totalSize;//总大小 public LongAdder finishedSize=new LongAdder();//本地已经下载大小 public double preSize;//前一次下载大小 public volatile LongAdder downSize=new LongAdder();//本次累计下载大小 public DownLoadInfoThread(long totalSize) { this.totalSize = totalSize; } @Override public void run() { //文件总大小(MB) String totalSize = String.format("%.2f", this.totalSize / MB); //当前秒下载大小(kb) int speed = (int) ((downSize.doubleValue() - preSize) / 1024d); //赋值当前size给之前得记录 preSize = downSize.doubleValue(); //计算当前剩余时间->转换为kb double remainSize = this.totalSize - finishedSize.doubleValue() - downSize.doubleValue(); //计算时间 String remainTime = String.format("%.1f", remainSize / (1024d * speed)); if ("infinity".equalsIgnoreCase(remainTime)) {//如果因为网速过慢导致文件下载时间为无限大，返回- remainTime = "-"; } //已下载 String currentSize = String.format("%.2f", (downSize.doubleValue() - finishedSize.doubleValue()) / MB); String info = String.format("已经下载%sMB/%sMB,速度%s/kb,剩余时间%ss", currentSize, totalSize, speed, remainTime); System.out.print("\r"); System.out.print(info+"------------"); } }
分片下载顾名思义需要先把文件拆解，拆解下载下来的文件是不能用的，所以需要合并分解出来的几个文件，然后将这几个临时文件合并然后删除掉,下面是下载器具体实现和拆分合并，删除临时文件的操作。
public class DownLoader { //日志线程 private ScheduledExecutorService executorService = Executors.newScheduledThreadPool(1); //分片下载线程 private ThreadPoolExecutor executorTask = new ThreadPoolExecutor(Thread_Num, Thread_Num, 0, TimeUnit.SECONDS, new ArrayBlockingQueue<>(Thread_Num)); //计数器 private CountDownLatch countDownLatch = new CountDownLatch(Thread_Num); /** * 下载器 * * @param url 下载地址 */ public void downLoad(String url) { HttpURLConnection httpURLConnection = null; try { //文件名 String fileName = HttpUtils.getFileName(url); //路径名 fileName = Down_Path.getPath() + fileName; //创建目标源链接 httpURLConnection = HttpUtils.httpURLConnection(url); //查看本地是否有该文件 long fileSize = FileUtils.getFileSize(fileName); //文件大小 int contentLength = httpURLConnection.getContentLength(); //判断文件是否下载 if (fileSize >= contentLength) { System.out.println("文件已经下载过了，请到该路径下查看" + fileName); return; } //创建文件info DownLoadInfoThread downLoadInfoThread = new DownLoadInfoThread(contentLength); //打印下载信息 executorService.scheduleAtFixedRate(downLoadInfoThread, 1, 1, TimeUnit.SECONDS); //分片 ArrayList<Future> list = new ArrayList<>(); splitFile(url, list, downLoadInfoThread,countDownLatch); countDownLatch.await(); //合并文件 if (merge(fileName)) { deleteFile(fileName); } } catch (IOException e) { throw new RuntimeException(e); } catch (InterruptedException e) { throw new RuntimeException(e); } finally { //关闭链接 httpURLConnection.disconnect(); executorTask.shutdown(); //关闭线程 executorService.shutdownNow(); System.out.print("\b"); System.out.print("-info-下载完成"); } } /** * 分片下载器 * * @param url 目标下载地址 * @param futureList 任务 * @param downLoadInfoThread * @param countDownLatch */ public void splitFile(String url, ArrayList<Future> futureList, DownLoadInfoThread downLoadInfoThread, CountDownLatch countDownLatch) { //文件大小 long fileLength = HttpUtils.getFileLength(url); //分片每个的大小 long size = fileLength / Thread_Num; //计算开始和结束位置 for (int i = 0; i < Thread_Num; i++) { long startPos = size * i; long endPos; if (i == Thread_Num - 1) {//当最后一个分片时，结束位置为文件大小 endPos = 0; } else { endPos = startPos + size; } if (startPos != 0) { startPos++; } //创建任务 DownLoadTask downLoadTask = new DownLoadTask(url, startPos, endPos, i, downLoadInfoThread,countDownLatch); Future<Boolean> submit = executorTask.submit(downLoadTask); futureList.add(submit); } } public boolean merge(String fileName) throws IOException { System.out.println(Thread.currentThread().getName() + "合并文件" + fileName); //创建输出流 int length = -1; byte[] buffer = new byte[1024 * 1000]; try ( RandomAccessFile accessFile = new RandomAccessFile(fileName, "rw"); ) { for (int i = 0; i < Thread_Num; i++) { //循环获取分片文件 BufferedInputStream bis = new BufferedInputStream(Files.newInputStream(Paths.get(fileName + ".temp" + i))); while ((length = bis.read(buffer)) != -1) { accessFile.write(buffer, 0, length); } } System.out.println("-success-合并成功" + fileName); } catch (Exception e) { System.out.println("合并失败"); return false; } return true; } public void deleteFile(String fileName) { for (int i = 0; i < Thread_Num; i++) { File file = new File(fileName + ".temp" + i); file.deleteOnExit(); System.out.println("删除临时文件" + fileName + "成功"); } } }
测试及最后效果
public class test { public static void main(String[] args) { DownLoader loader = new DownLoader(); loader.downLoad("https://dldir1.qq.com/qqfile/qq/PCQQ9.7.17/QQ9.7.17.29230.exe"); } }
3461
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
