Java 多线程断点下载文件

最新推荐文章于 2024-07-24 16:04:04 发布

hoojo

最新推荐文章于 2024-07-24 16:04:04 发布

阅读量2.8k

点赞数 2

分类专栏： JavaSE JavaEE 文章标签：多线程 java string url download bean

本文链接：https://blog.csdn.net/ibm_hoojo/article/details/6838222

版权

JavaEE 同时被 2 个专栏收录

100 篇文章 1 订阅

订阅专栏

JavaSE

33 篇文章 0 订阅

订阅专栏

基本原理：利用URLConnection获取要下载文件的长度、头部等相关信息，并设置响应的头部信息。并且通过URLConnection获取输入流，将文件分成指定的块，每一块单独开辟一个线程完成数据的读取、写入。通过输入流读取下载文件的信息，然后将读取的信息用RandomAccessFile随机写入到本地文件中。同时，每个线程写入的数据都文件指针也就是写入数据的长度，需要保存在一个临时文件中。这样当本次下载没有完成的时候，下次下载的时候就从这个文件中读取上一次下载的文件长度，然后继续接着上一次的位置开始下载。并且将本次下载的长度写入到这个文件中。

个人博客：

http://hoojo.cnblogs.com

http://blog.csdn.net/IBM_hoojo

email: hoojo_@126.com

一、下载文件信息类、实体

封装即将下载资源的信息

package com.hoo.entity;

/**
 * <b>function:</b> 下载文件信息类
 * @author hoojo
 * @createDate 2011-9-21 下午05:14:58
 * @file DownloadInfo.java
 * @package com.hoo.entity
 * @project MultiThreadDownLoad
 * @blog http://blog.csdn.net/IBM_hoojo
 * @email hoojo_@126.com
 * @version 1.0
 */
public class DownloadInfo {
	//下载文件url
	private String url;
	//下载文件名称
	private String fileName;
	//下载文件路径
	private String filePath;
	//分成多少段下载， 每一段用一个线程完成下载
	private int splitter;
	
	//下载文件默认保存路径
	private final static String FILE_PATH = "C:/temp";
	//默认分块数、线程数
	private final static int SPLITTER_NUM = 5;
	
	public DownloadInfo() {
		super();
	}
	
	/**
	 * @param url 下载地址
	 */
	public DownloadInfo(String url) {
		this(url, null, null, SPLITTER_NUM);
	}
	
	/**
	 * @param url 下载地址url
	 * @param splitter 分成多少段或是多少个线程下载
	 */
	public DownloadInfo(String url, int splitter) {
		this(url, null, null, splitter);
	}
	
	/***
	 * @param url 下载地址
	 * @param fileName 文件名称
	 * @param filePath 文件保存路径
	 * @param splitter 分成多少段或是多少个线程下载
	 */
	public DownloadInfo(String url, String fileName, String filePath, int splitter) {
		super();
		if (url == null || "".equals(url)) {
			throw new RuntimeException("url is not null!");
		}
		this.url =  url;
		this.fileName = (fileName == null || "".equals(fileName)) ? getFileName(url) : fileName;
		this.filePath = (filePath == null || "".equals(filePath)) ? FILE_PATH : filePath;
		this.splitter = (splitter < 1) ? SPLITTER_NUM : splitter;
	}
	
	/**
	 * <b>function:</b> 通过url获得文件名称
	 * @author hoojo
	 * @createDate 2011-9-30 下午05:00:00
	 * @param url
	 * @return
	 */
	private String getFileName(String url) {
		return url.substring(url.lastIndexOf("/") + 1, url.length());
	}
	
	public String getUrl() {
		return url;
	}

	public void setUrl(String url) {
		if (url == null || "".equals(url)) {
			throw new RuntimeException("url is not null!");
		}
		this.url = url;
	}

	public String getFileName() {
		return fileName;
	}

	public void setFileName(String fileName) {
		this.fileName = (fileName == null || "".equals(fileName)) ? getFileName(url) : fileName;
	}

	public String getFilePath() {
		return filePath;
	}

	public void setFilePath(String filePath) {
		this.filePath = (filePath == null || "".equals(filePath)) ? FILE_PATH : filePath;
	}

	public int getSplitter() {
		return splitter;
	}

	public void setSplitter(int splitter) {
		this.splitter = (splitter < 1) ? SPLITTER_NUM : splitter;
	}
	
	@Override
	public String toString() {
		return this.url + "#" + this.fileName + "#" + this.filePath + "#" + this.splitter;
	}
}

二、随机写入一段文件

package com.hoo.download;

import java.io.IOException;
import java.io.RandomAccessFile;

/**
 * <b>function:</b> 写入文件、保存文件
 * @author hoojo
 * @createDate 2011-9-21 下午05:44:02
 * @file SaveItemFile.java
 * @package com.hoo.download
 * @project MultiThreadDownLoad
 * @blog http://blog.csdn.net/IBM_hoojo
 * @email hoojo_@126.com
 * @version 1.0
 */
public class SaveItemFile {
	//存储文件
	private RandomAccessFile itemFile;
	
	public SaveItemFile() throws IOException {
		this("", 0);
	}
	
	/**
	 * @param name 文件路径、名称
	 * @param pos 写入点位置 position
	 * @throws IOException
	 */
	public SaveItemFile(String name, long pos) throws IOException {
		itemFile = new RandomAccessFile(name, "rw");
		//在指定的pos位置开始写入数据
		itemFile.seek(pos);
	}
	
	/**
	 * <b>function:</b> 同步方法写入文件
	 * @author hoojo
	 * @createDate 2011-9-26 下午12:21:22
	 * @param buff 缓冲数组
	 * @param start 起始位置
	 * @param length 长度
	 * @return
	 */
	public synchronized int write(byte[] buff, int start, int length) {
		int i = -1;
		try {
			itemFile.write(buff, start, length);
			i = length;
		} catch (IOException e) {
			e.printStackTrace();
		}
		return i;
	}
	
	public void close() throws IOException {
		if (itemFile != null) {
			itemFile.close();
		}
	}
}

这个类主要是完成向本地的指定文件指针出开始写入文件，并返回当前写入文件的长度（文件指针）。这个类将被线程调用，文件被分成对应的块后，将被线程调用。每个线程都将会调用这个类完成文件的随机写入。

三、单个线程下载文件

package com.hoo.download;

import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import com.hoo.util.LogUtils;

/**
 * <b>function:</b> 单线程下载文件
 * @author hoojo
 * @createDate 2011-9-22 下午02:55:10
 * @file DownloadFile.java
 * @package com.hoo.download
 * @project MultiThreadDownLoad
 * @blog http://blog.csdn.net/IBM_hoojo
 * @email hoojo_@126.com
 * @version 1.0
 */
public class DownloadFile extends Thread {
	
	//下载文件url
	private String url;
	//下载文件起始位置  
	private long startPos;
	//下载文件结束位置
	private long endPos;
	//线程id
	private int threadId;
	
	//下载是否完成
	private boolean isDownloadOver = false;

	private SaveItemFile itemFile;
	
	private static final int BUFF_LENGTH = 1024 * 8;
	
	/**
	 * @param url 下载文件url
	 * @param name 文件名称
	 * @param startPos 下载文件起点
	 * @param endPos 下载文件结束点
	 * @param threadId 线程id
	 * @throws IOException
	 */
	public DownloadFile(String url, String name, long startPos, long endPos, int threadId) throws IOException {
		super();
		this.url = url;
		this.startPos = startPos;
		this.endPos = endPos;
		this.threadId = threadId;
		//分块下载写入文件内容
		this.itemFile = new SaveItemFile(name, startPos);
	}

	
	@Override
	public void run() {
		while (endPos > startPos && !isDownloadOver) {
			try {
				URL url = new URL(this.url);
				HttpURLConnection conn = (HttpURLConnection) url.openConnection();
				
				// 设置连接超时时间为10000ms
				conn.setConnectTimeout(10000);
				// 设置读取数据超时时间为10000ms
				conn.setReadTimeout(10000);
				
				setHeader(conn);
				
				String property = "bytes=" + startPos + "-";
				conn.setRequestProperty("RANGE", property);
				
				//输出log信息
				LogUtils.log("开始 " + threadId + "：" + property + endPos);
				//printHeader(conn);
				
				//获取文件输入流，读取文件内容
				InputStream is = conn.getInputStream();
				
				byte[] buff = new byte[BUFF_LENGTH];
				int length = -1;
				LogUtils.log("#start#Thread: " + threadId + ", startPos: " + startPos + ", endPos: " + endPos);
				while ((length = is.read(buff)) > 0 && startPos < endPos && !isDownloadOver) {
					//写入文件内容，返回最后写入的长度
					startPos += itemFile.write(buff, 0, length);
				}
				LogUtils.log("#over#Thread: " + threadId + ", startPos: " + startPos + ", endPos: " + endPos);
				LogUtils.log("Thread " + threadId + " is execute over!");
				this.isDownloadOver = true;
			} catch (MalformedURLException e) {
				e.printStackTrace();
			} catch (IOException e) {
				e.printStackTrace();
			} finally {
				try {
					if (itemFile != null) {
						itemFile.close();
					}
				} catch (IOException e) {
					e.printStackTrace();
				}
			}
		}
		if (endPos < startPos && !isDownloadOver) {
			LogUtils.log("Thread " + threadId  + " startPos > endPos, not need download file !");
			this.isDownloadOver = true;
		}
		if (endPos == startPos && !isDownloadOver) {
			LogUtils.log("Thread " + threadId  + " startPos = endPos, not need download file !");
			this.isDownloadOver = true;
		}
	}
	
	/**
	 * <b>function:</b> 打印下载文件头部信息
	 * @author hoojo
	 * @createDate 2011-9-22 下午05:44:35
	 * @param conn HttpURLConnection
	 */
	public static void printHeader(URLConnection conn) {
		int i = 1;
		while (true) {
			String header = conn.getHeaderFieldKey(i);
			i++;
			if (header != null) {
				LogUtils.info(header + ":" + conn.getHeaderField(i));
			} else {
				break;
			}
		}
	}
	
	/**
	 * <b>function:</b> 设置URLConnection的头部信息，伪装请求信息
	 * @author hoojo
	 * @createDate 2011-9-28 下午05:29:43
	 * @param con
	 */
	public static void setHeader(URLConnection conn) {
		conn.setRequestProperty("User-Agent", "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.3) Gecko/2008092510 Ubuntu/8.04 (hardy) Firefox/3.0.3");
		conn.setRequestProperty("Accept-Language", "en-us,en;q=0.7,zh-cn;q=0.3");
		conn.setRequestProperty("Accept-Encoding", "utf-8");
		conn.setRequestProperty("Accept-Charset", "ISO-8859-1,utf-8;q=0.7,*;q=0.7");
		conn.setRequestProperty("Keep-Alive", "300");
		conn.setRequestProperty("connnection", "keep-alive");
		conn.setRequestProperty("If-Modified-Since", "Fri, 02 Jan 2009 17:00:05 GMT");
		conn.setRequestProperty("If-None-Match", "\"1261d8-4290-df64d224\"");
		conn.setRequestProperty("Cache-conntrol", "max-age=0");
		conn.setRequestProperty("Referer", "http://www.baidu.com");
	}
	
	public boolean isDownloadOver() {
		return isDownloadOver;
	}
	
	public long getStartPos() {
		return startPos;
	}

	public long getEndPos() {
		return endPos;
	}
}

这个类主要是完成单个线程的文件下载，将通过URLConnection读取指定url的资源信息。然后用InputStream读取文件内容，然后调用调用SaveItemFile类，向本地写入当前要读取的块的内容。

四、分段多线程写入文件内容

package com.hoo.download;

import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import com.hoo.entity.DownloadInfo;
import com.hoo.util.LogUtils;

/**
 * <b>function:</b> 分批量下载文件
 * @author hoojo
 * @createDate 2011-9-22 下午05:51:54
 * @file BatchDownloadFile.java
 * @package com.hoo.download
 * @project MultiThreadDownLoad
 * @blog http://blog.csdn.net/IBM_hoojo
 * @email hoojo_@126.com
 * @version 1.0
 */
public class BatchDownloadFile implements Runnable {
	//下载文件信息 
	private DownloadInfo downloadInfo;
	//一组开始下载位置
	private long[] startPos;
	//一组结束下载位置
	private long[] endPos;
	//休眠时间
	private static final int SLEEP_SECONDS = 500;
	//子线程下载
	private DownloadFile[] fileItem;
	//文件长度
	private int length;
	//是否第一个文件
	private boolean first = true;
	//是否停止下载
	private boolean stop = false;
	//临时文件信息
	private File tempFile;
	
	public BatchDownloadFile(DownloadInfo downloadInfo) {
		this.downloadInfo = downloadInfo;
		String tempPath = this.downloadInfo.getFilePath() + File.separator + downloadInfo.getFileName() + ".position";
		tempFile = new File(tempPath);
		//如果存在读入点位置的文件
		if (tempFile.exists()) {
			first = false;
			//就直接读取内容
			try {
				readPosInfo();
			} catch (IOException e) {
				e.printStackTrace();
			}
		} else {
			//数组的长度就要分成多少段的数量
			startPos = new long[downloadInfo.getSplitter()];
			endPos = new long[downloadInfo.getSplitter()];
		}
	}
	
	@Override
	public void run() {
		//首次下载，获取下载文件长度
		if (first) {
			length = this.getFileSize();//获取文件长度
			if (length == -1) {
				LogUtils.log("file length is know!");
				stop = true;
			} else if (length == -2) {
				LogUtils.log("read file length is error!");
				stop = true;
			} else if (length > 0) {
				/**
				 * eg 
				 * start: 1, 3, 5, 7, 9
				 * end: 3, 5, 7, 9, length
				 */
				for (int i = 0, len = startPos.length; i < len; i++) {
					int size = i * (length / len);
					startPos[i] = size;
					
					//设置最后一个结束点的位置
					if (i == len - 1) {
						endPos[i] = length;
					} else {
						size = (i + 1) * (length / len);
						endPos[i] = size;
					}
					LogUtils.log("start-end Position[" + i + "]: " + startPos[i] + "-" + endPos[i]);
				}
			} else {
				LogUtils.log("get file length is error, download is stop!");
				stop = true;
			}
		}
		
		//子线程开始下载
		if (!stop) {
			//创建单线程下载对象数组
			fileItem = new DownloadFile[startPos.length];//startPos.length = downloadInfo.getSplitter()
			for (int i = 0; i < startPos.length; i++) {
				try {
					//创建指定个数单线程下载对象，每个线程独立完成指定块内容的下载
					fileItem[i] = new DownloadFile(
						downloadInfo.getUrl(), 
						this.downloadInfo.getFilePath() + File.separator + downloadInfo.getFileName(), 
						startPos[i], endPos[i], i
					);
					fileItem[i].start();//启动线程，开始下载
					LogUtils.log("Thread: " + i + ", startPos: " + startPos[i] + ", endPos: " + endPos[i]);
				} catch (IOException e) {
					e.printStackTrace();
				}
			}
			
			//循环写入下载文件长度信息
			while (!stop) {
				try {
					writePosInfo();
					LogUtils.log("downloading……");
					Thread.sleep(SLEEP_SECONDS);
					stop = true;
				} catch (IOException e) {
					e.printStackTrace();
				} catch (InterruptedException e) {
					e.printStackTrace();
				}
				for (int i = 0; i < startPos.length; i++) {
					if (!fileItem[i].isDownloadOver()) {
						stop = false;
						break;
					}
				}
			}
			LogUtils.info("Download task is finished!");
		}
	}
	
	/**
	 * 将写入点数据保存在临时文件中
	 * @author hoojo
	 * @createDate 2011-9-23 下午05:25:37
	 * @throws IOException
	 */
	private void writePosInfo() throws IOException {
		DataOutputStream dos = new DataOutputStream(new FileOutputStream(tempFile));
		dos.writeInt(startPos.length);
		for (int i = 0; i < startPos.length; i++) {
			dos.writeLong(fileItem[i].getStartPos());
			dos.writeLong(fileItem[i].getEndPos());
			//LogUtils.info("[" + fileItem[i].getStartPos() + "#" + fileItem[i].getEndPos() + "]");
		}
		dos.close();
	}
	
	/**
	 * <b>function:</b>读取写入点的位置信息
	 * @author hoojo
	 * @createDate 2011-9-23 下午05:30:29
	 * @throws IOException
	 */
	private void readPosInfo() throws IOException {
		DataInputStream dis = new DataInputStream(new FileInputStream(tempFile));
		int startPosLength = dis.readInt();
		startPos = new long[startPosLength];
		endPos = new long[startPosLength];
		for (int i = 0; i < startPosLength; i++) {
			startPos[i] = dis.readLong();
			endPos[i] = dis.readLong();
		}
		dis.close();
	}
	
	/**
	 * <b>function:</b> 获取下载文件的长度
	 * @author hoojo
	 * @createDate 2011-9-26 下午12:15:08
	 * @return
	 */
	private int getFileSize() {
		int fileLength = -1;
		try {
			URL url = new URL(this.downloadInfo.getUrl());
			HttpURLConnection conn = (HttpURLConnection) url.openConnection();
			
			DownloadFile.setHeader(conn);

			int stateCode = conn.getResponseCode();
			//判断http status是否为HTTP/1.1 206 Partial Content或者200 OK
			if (stateCode != HttpURLConnection.HTTP_OK && stateCode != HttpURLConnection.HTTP_PARTIAL) {
				LogUtils.log("Error Code: " + stateCode);
				return -2;
			} else if (stateCode >= 400) {
				LogUtils.log("Error Code: " + stateCode);
				return -2;
			} else {
				//获取长度
				fileLength = conn.getContentLength();
				LogUtils.log("FileLength: " + fileLength);
			}
			
			//读取文件长度
			/*for (int i = 1; ; i++) {
				String header = conn.getHeaderFieldKey(i);
				if (header != null) {
					if ("Content-Length".equals(header)) {
						fileLength = Integer.parseInt(conn.getHeaderField(i));
						break;
					}
				} else {
					break;
				}
			}
			*/
			
			DownloadFile.printHeader(conn);
		} catch (MalformedURLException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		}
		return fileLength;
	}
}

这个类主要是完成读取指定url资源的内容，获取该资源的长度。然后将该资源分成指定的块数，将每块的起始下载位置、结束下载位置，分别保存在一个数组中。每块都单独开辟一个独立线程开始下载。在开始下载之前，需要创建一个临时文件，写入当前下载线程的开始下载指针位置和结束下载指针位置。

五、工具类、测试类

日志工具类

package com.hoo.util;

/**
 * <b>function:</b> 日志工具类
 * @author hoojo
 * @createDate 2011-9-21 下午05:21:27
 * @file LogUtils.java
 * @package com.hoo.util
 * @project MultiThreadDownLoad
 * @blog http://blog.csdn.net/IBM_hoojo
 * @email hoojo_@126.com
 * @version 1.0
 */
public abstract class LogUtils {
	
	public static void log(Object message) {
		System.err.println(message);
	}
	
	public static void log(String message) {
		System.err.println(message);
	}
	
	public static void log(int message) {
		System.err.println(message);
	}
	
	public static void info(Object message) {
		System.out.println(message);
	}
	
	public static void info(String message) {
		System.out.println(message);
	}
	
	public static void info(int message) {
		System.out.println(message);
	}
}

下载工具类

package com.hoo.util;

import com.hoo.download.BatchDownloadFile;
import com.hoo.entity.DownloadInfo;

/**
 * <b>function:</b> 分块多线程下载工具类
 * @author hoojo
 * @createDate 2011-9-28 下午05:22:18
 * @file DownloadUtils.java
 * @package com.hoo.util
 * @project MultiThreadDownLoad
 * @blog http://blog.csdn.net/IBM_hoojo
 * @email hoojo_@126.com
 * @version 1.0
 */
public abstract class DownloadUtils {

	public static void download(String url) {
		DownloadInfo bean = new DownloadInfo(url);
		LogUtils.info(bean);
		BatchDownloadFile down = new BatchDownloadFile(bean);
		new Thread(down).start();
	}
	
	public static void download(String url, int threadNum) {
		DownloadInfo bean = new DownloadInfo(url, threadNum);
		LogUtils.info(bean);
		BatchDownloadFile down = new BatchDownloadFile(bean);
		new Thread(down).start();
	}
	
	public static void download(String url, String fileName, String filePath, int threadNum) {
		DownloadInfo bean = new DownloadInfo(url, fileName, filePath, threadNum);
		LogUtils.info(bean);
		BatchDownloadFile down = new BatchDownloadFile(bean);
		new Thread(down).start();
	}
}

下载测试类

package com.hoo.test;

import com.hoo.util.DownloadUtils;

/**
 * <b>function:</b> 下载测试
 * @author hoojo
 * @createDate 2011-9-23 下午05:49:46
 * @file TestDownloadMain.java
 * @package com.hoo.download
 * @project MultiThreadDownLoad
 * @blog http://blog.csdn.net/IBM_hoojo
 * @email hoojo_@126.com
 * @version 1.0
 */
public class TestDownloadMain {

	public static void main(String[] args) {
		/*DownloadInfo bean = new DownloadInfo("http://i7.meishichina.com/Health/UploadFiles/201109/2011092116224363.jpg");
		System.out.println(bean);
		BatchDownloadFile down = new BatchDownloadFile(bean);
		new Thread(down).start();*/
		
		//DownloadUtils.download("http://i7.meishichina.com/Health/UploadFiles/201109/2011092116224363.jpg");
		DownloadUtils.download("http://mp3.baidu.com/j?j=2&url=http%3A%2F%2Fzhangmenshiting2.baidu.com%2Fdata%2Fmusic%2F1669425%2F%25E9%2599%25B7%25E5%2585%25A5%25E7%2588%25B1%25E9%2587%258C%25E9%259D%25A2.mp3%3Fxcode%3D2ff36fb70737c816553396c56deab3f1", "aa.mp3", "c:/temp", 5);
	}
}

多线程下载主要在第三部和第四部，其他的地方还是很好理解。源码中提供相应的注释了，便于理解。