import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.RandomAccessFile;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
public class DownloadTask {
private static final Logger logger = Logger.getLogger(DownloadTask.class);
// 分段下载的线程个数
private int threadNum = 5;
private URL url;
private long threadLength = 0L;
// 目标文件路径与名字
public String fileDir;
public String fileName;
public boolean statusError = false;
public long sleepSeconds = 5;
private boolean isExistAndDone = false;
public DownloadTask(String fileDir, String fileName) {
this.fileDir = fileDir;
this.fileName = fileName;
}
public String download(String urlStr) throws IOException {
statusError = false;
long contentLength = 0;
CountDownLatch latch = new CountDownLatch(threadNum);
ChildThread[] childThreads = new ChildThread[threadNum];
long[] startPos = new long[threadNum];
long[] endPos = new long[threadNum];
File file = new File(fileDir + fileName);
File tempFile = new File(fileDir + fileName + "_temp");
this.url = new URL(urlStr);
HttpURLConnection con = (HttpURLConnection) url.openConnection();
// 设置连接超时时间为30000ms
con.setConnectTimeout(30000);
// 设置读取数据超时时间为30000ms
con.setReadTimeout(30000);
setHeader(con);
if (con.getResponseCode() != HttpURLConnection.HTTP_OK
&& con.getResponseCode() != HttpURLConnection.HTTP_PARTIAL) {
return "httpcode_" + (con.getResponseCode() + "_" + StringUtils.replace(con.getResponseMessage(), " ", "_"));
}
// 得到content的长度
contentLength = con.getContentLength();
// 把context分为threadNum段的话,每段的长度。
this.threadLength = contentLength / threadNum;
// 第一步,分析已下载的临时文件,设置断点,如果是新的下载任务,则建立目标文件。
setThreadBreakpoint(file, tempFile, contentLength, startPos, endPos);
if (isExistAndDone) {
return fileName + ": " + (contentLength / 1024) + "K";
}
// 第二步,分多个线程下载文件
ExecutorService exec = Executors.newCachedThreadPool();
for (int i = 0; i < threadNum; i++) {
// 开启子线程,并执行。
ChildThread thread = new ChildThread(this, latch, i, startPos[i], endPos[i]);
childThreads[i] = thread;
exec.execute(thread);
}
try {
// 等待CountdownLatch信号为0,表示所有子线程都结束。
latch.await();
exec.shutdown();
// 删除临时文件
long downloadFileSize = file.length();
if (downloadFileSize == contentLength) {
logger.info("delete temp file: " + tempFile.getName());
tempFile.delete();
}
} catch (InterruptedException e) {
e.printStackTrace();
}
return fileName + ": " + (contentLength / 1024) + "K";
}
private void setThreadBreakpoint(File file, File tempFile, long contentLength, long[] startPos, long[] endPos) throws IOException {
RandomAccessFile tempFileFos = null;
try {
if (file.exists()) {
logger.info("file " + fileName + " has exists!");
long localFileSize = file.length();
// 下载的目标文件已存在,判断目标文件是否完整
if (localFileSize < contentLength) {
logger.info("Now download continue ... ");
tempFileFos = new RandomAccessFile(tempFile, "rw");
// 遍历目标文件的所有临时文件,设置断点的位置,即每个临时文件的长度
for (int i = 0; i < threadNum; i++) {
tempFileFos.seek(4 + 24 * i + 8);
endPos[i] = tempFileFos.readLong();
tempFileFos.seek(4 + 24 * i + 16);
startPos[i] = tempFileFos.readLong();
}
} else {
isExistAndDone = true;
logger.info("This file has download complete!");
}
} else {
// 如果下载的目标文件不存在,则创建新文件
file.createNewFile();
tempFile.createNewFile();
tempFileFos = new RandomAccessFile(tempFile, "rw");
tempFileFos.writeInt(threadNum);
for (int i = 0; i < threadNum; i++) {
// 创建子线程来负责下载数据,每段数据的起始位置为(threadLength * i)
startPos[i] = threadLength * i;
tempFileFos.writeLong(startPos[i]);
/*
* 设置子线程的终止位置,非最后一个线程即为(threadLength * (i + 1) - 1)
* 最后一个线程的终止位置即为下载内容的长度
*/
if (i == threadNum - 1) {
endPos[i] = contentLength;
} else {
endPos[i] = threadLength * (i + 1) - 1;
}
// end position
tempFileFos.writeLong(endPos[i]);
// current position
tempFileFos.writeLong(startPos[i]);
}
}
} finally {
tempFileFos.close();
}
}
public class ChildThread extends Thread {
private DownloadTask task;
private int id;
private long startPosition;
private long endPosition;
private final CountDownLatch latch;
private RandomAccessFile file = null;
private RandomAccessFile tempFile = null;
public ChildThread(DownloadTask task, CountDownLatch latch, int id, long startPos, long endPos) throws FileNotFoundException {
super();
this.task = task;
this.id = id;
this.startPosition = startPos;
this.endPosition = endPos;
this.latch = latch;
file = new RandomAccessFile(this.task.fileDir + this.task.fileName, "rw");
tempFile = new RandomAccessFile(this.task.fileDir + this.task.fileName + "_temp", "rw");
}
@Override
public void run() {
logger.info("Thread " + id + " run ...");
HttpURLConnection con = null;
InputStream inputStream = null;
long count = 0;
try {
tempFile.seek(4 + 24 * id);
} catch (IOException e2) {
e2.printStackTrace();
}
for (;;) {
try {
// 打开URLConnection
con = (HttpURLConnection) task.url.openConnection();
setHeader(con);
// 设置连接超时时间为30000ms
con.setConnectTimeout(30000);
// 设置读取数据超时时间为30000ms
con.setReadTimeout(30000);
if (startPosition < endPosition) {
// 设置下载数据的起止区间
con.setRequestProperty("Range", "bytes="
+ startPosition + "-" + endPosition);
logger.info("Thread " + id
+ " startPosition is " + startPosition
+ ", and endPosition is " + endPosition);
file.seek(startPosition);
// 判断http status是否为HTTP/1.1 206 Partial Content或者200 OK
// 如果不是以上两种状态,把status改为STATUS_HTTPSTATUS_ERROR
if (con.getResponseCode() != HttpURLConnection.HTTP_OK
&& con.getResponseCode() != HttpURLConnection.HTTP_PARTIAL) {
logger.info("Thread " + id + ": code = "
+ con.getResponseCode() + ", status = "
+ con.getResponseMessage());
this.task.statusError = true;
file.close();
con.disconnect();
logger.info("Thread " + id + " finished.");
latch.countDown();
break;
}
inputStream = con.getInputStream();
int len = 0;
byte[] b = new byte[1024];
while (!this.task.statusError
&& (len = inputStream.read(b)) != -1) {
file.write(b, 0, len);
count += len;
startPosition += len;
// set tempFile now position
tempFile.seek(4 + 24 * id + 16);
tempFile.writeLong(startPosition);
}
file.close();
tempFile.close();
inputStream.close();
con.disconnect();
}
logger.info("Thread " + id + " finished.");
latch.countDown();
break;
} catch (IOException e) {
try {
// outputStream.flush();
TimeUnit.SECONDS.sleep(sleepSeconds);
} catch (InterruptedException e1) {
e1.printStackTrace();
}
continue;
}
}
}
}
private void setHeader(URLConnection con) {
con.setRequestProperty("User-Agent", "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.3) Gecko/2008092510 Ubuntu/8.04 (hardy) Firefox/3.0.3");
}
}
多线程并行下载,断点续传
最新推荐文章于 2024-04-30 14:55:09 发布