1.文件操作类,负责文件写入:
import java.io.IOException;
import java.io.RandomAccessFile;
import java.io.Serializable;
public class FileAccess implements Serializable {
private RandomAccessFile oSavedFile;
private long nPos;
public FileAccess() throws IOException {
this("",0);
}
public FileAccess(String sName, long nPos) throws IOException {
this.oSavedFile = new RandomAccessFile(sName, "rw");
this.nPos = nPos;
this.oSavedFile.seek(nPos);
}
//不需要同步,写得位置不一样
public int write(byte[] b, int nStart, int nLen) {
int n = -1;
try {
this.oSavedFile.write(b,nStart,nLen);
n = nLen;
} catch(IOException e) {
e.printStackTrace();
}
return n;
}
}
2.文件下载器,负责启动 多线程下载:
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URL;
public class FileDownloader {
//文件信息Bean
private SiteInfoBean siteInfoBean = null;
//多个线程
private long[] nStartPos;//开始位置
private long[] nEndPos;//结束位置
//子线程对象
private FileSplitterFetch[] fileSplitterFetch;
//文件长度
private long nFileLength;
//是否第一次读取文件
private boolean bFirst = true;
//停止标志
//boolean bStop = false;
//文件下载的临时信息,保存下载信息
private File tmpFile;
//输出到文件的输出流
private DataOutputStream output;
//下载完成的线程数量
private int count;
public FileDownloader() {
}
public FileDownloader(SiteInfoBean bean) {
// TODO Auto-generated constructor stub
this.siteInfoBean = bean;
this.tmpFile = new File(bean.getsFilePath()+File.separator+bean.getsFileName()+".info");
System.out.println(tmpFile.getAbsolutePath());
if(this.tmpFile.exists()) {
this.bFirst = false;
//读取下载信息
read_pos();
} else {
//splitter 文件被分割的份数
this.nStartPos = new long[this.siteInfoBean.getnSplitter()];
this.nEndPos = new long[this.siteInfoBean.getnSplitter()];
}
}
public void read_pos() {
// TODO Auto-generated method stub
try {
DataInputStream input = new DataInputStream(new FileInputStream(tmpFile));
//第一个数字,线程数量
int nCount = input.readInt();
this.nStartPos = new long[nCount];
this.nEndPos = new long[nCount];
for(int i=0;i<this.nStartPos.length;i++) {
this.nStartPos[i] = input.readLong();
this.nEndPos[i] = input.readLong();
}
input.close();
} catch(Exception e) {
e.printStackTrace();
}
}
public void download() throws IOException, InterruptedException {
//如果是第一次下载,没有info文件,则分配复制 起始 端点下载量
if(this.bFirst) {
//获取文件长度
this.nFileLength = getFileSize();
System.out.println("------------fileLen-----------------"+this.nFileLength);
if(this.nFileLength==-1) {
System.err.println("File length is not know");
} else if(this.nFileLength==-2){
System.err.println("File is not access!");
} else {
//block开头
for(int i=0;i<this.nStartPos.length;i++) {
//nStartPos.length是自己规定的
this.nStartPos[i] = (long)(i*(this.nFileLength/this.nStartPos.length));
}
//block末尾
for(int i=0;i<this.nEndPos.length-1;i++) {
this.nEndPos[i] = this.nStartPos[i+1];
}
this.nEndPos[this.nEndPos.length-1] = this.nFileLength;
//分配好之后,创建断点记录文件
try {
this.output = new DataOutputStream(new FileOutputStream(tmpFile));
} catch (FileNotFoundException e) {
e.printStackTrace();
}
}
}
//启动子线程
this.fileSplitterFetch = new FileSplitterFetch[this.nStartPos.length];
System.out.println("---------------------------------");
for(int i=0;i<this.nStartPos.length;i++) {
this.fileSplitterFetch[i] = new FileSplitterFetch(this.siteInfoBean.getsSiteURL(),
this.siteInfoBean.getsFilePath()+File.separator+this.siteInfoBean.getsFileName(),
this.nStartPos[i],this.nEndPos[i],i);
System.out.println("Thread "+i+" , nStartPos= "+this.nStartPos[i]+" , nEndPos="+this.nEndPos[i]);
this.fileSplitterFetch[i].start();
}
//是否结束while循环
boolean breakWhile = false;
while(!breakWhile) {
//
System.out.println("开始while。。。。。。。。");
//write_nPos();
//Utility.sleep(1000);
//Thread.sleep(1000);
//断点 记录
write_nPos();
//必须sleep,否则记录文件会被清空,没有数据 0kb
Utility.sleep(500);
breakWhile = true;
/*---------------------+++++++++++++++++++++++++++*/
for(int i=0;i<this.nStartPos.length;i++) {
//System.out.println("i++++++++++++++++++++++++++++++++++"+i);
//还没下载完成,检查如果有一个没有下载完就设置 breakWhile=false
if(!this.fileSplitterFetch[i].bDownOver) {
breakWhile = false;
break;
//单个线程 下载完成
} else if(!this.fileSplitterFetch[i].bStop) {
System.out.println("---------ok------------------"+"threadID-------------------"+i+this.fileSplitterFetch[i].bStop);
//停止线程
this.fileSplitterFetch[i].splitterStop();
System.out.println("---------ok------------------"+"threadID-------------------"+i+this.fileSplitterFetch[i].bStop);
count++;
System.out.println("-------------------------count-----------------"+count);
//如果5个线程都下载完成,停止while.
//有bug,如果接着上次的下载,count又重新计数了,没有意义了
if(count>=this.fileSplitterFetch.length) breakWhile = true;
}
}
System.out.println("break while---------------------------"+breakWhile);
//breakWhile=false如果没有下载完成,各个线程
if(breakWhile) {
System.out.println("文件下载结束!");
for(int i=0;i<nStartPos.length;i++)
fileSplitterFetch[i].splitterStop();
break;
}
//Thread.sleep(1000);
}
}
//保存下载信息(文件指针位置)
private void write_nPos() {
// TODO Auto-generated method stub
try {
this.output = new DataOutputStream(new FileOutputStream(tmpFile));
//下载线程数量
this.output.writeInt(this.nStartPos.length);
for(int i=0;i<this.nStartPos.length;i++) {
this.output.writeLong(this.fileSplitterFetch[i].nStartPos);
this.output.writeLong(this.fileSplitterFetch[i].nEndPos);
}
this.output.flush();
this.output.close();
} catch(Exception e) {
e.printStackTrace();
}
}
//获得文件长度
private long getFileSize() {
// TODO Auto-generated method stub
int nFileLength = -1;
try {
URL url = new URL(this.siteInfoBean.getsSiteURL());
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setRequestProperty("User-Agent",
"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.3) Gecko/2008092510 Ubuntu/8.04 (hardy) Firefox/3.0.3");
int responseCode = conn.getResponseCode();
if(responseCode>=400) {
processErrorCode(responseCode);
return -2; //-2 represents access is error
}
/*String sHeader;
for(int i=1; ; i++) {
sHeader = conn.getHeaderFieldKey(i);
if(sHeader!=null) {
if(sHeader.equals("Content-Length")) {
nFileLength = Integer.parseInt(conn.getHeaderField(sHeader));
break;
}
} else {
break;
}
}*/
nFileLength = conn.getContentLength();
} catch(Exception e) {
e.printStackTrace();
}
return nFileLength;
}
private void processErrorCode(int responseCode) {
// TODO Auto-generated method stub
System.err.println("Error Code: "+responseCode);
}
public static void main(String args[]) throws IOException, InterruptedException {
SiteInfoBean bean = new SiteInfoBean("http://127.0.0.1:8080/fileTest/download/a.zip","d:/test","a.zip",1);
//SiteInfoBean bean1 = new SiteInfoBean("http://localhost/download/friends.rmvb","C:\\xampp\\htdocs\\test","hello.rmvb",5);
FileDownloader downloader = new FileDownloader(bean);
downloader.download();
}
}
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URL;
/**
* 负责部分文件的抓取, 多线程
* @author ETHAN
*
*/
public class FileSplitterFetch extends Thread {
private String sURL; //file url
long nStartPos;//file snippet start position
long nEndPos; //file snippet end position
private int nThreadID;
boolean bDownOver = false;//downing is over;
boolean bStop = false; //stop identical相同
private FileAccess fileAccess = null; // file access interface
public FileSplitterFetch(String sURL,String sName,long nStart,long nEnd,int id) throws IOException {
this.sURL = sURL;
this.nStartPos = nStart;
this.nEndPos = nEnd;
this.nThreadID = id;
this.fileAccess = new FileAccess(sName,this.nStartPos);
}
public void run() {
while(this.nStartPos<this.nEndPos&&!this.bStop) {
System.out.println("-----------------------------------"+this.nThreadID+"---------开始写数据");
try {
URL url = new URL(sURL);
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setRequestProperty("User-Agent",
"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.3) Gecko/2008092510 Ubuntu/8.04 (hardy) Firefox/3.0.3");
String sProperty = "bytes="+this.nStartPos+"-";
conn.setRequestProperty("range", sProperty);
Utility.log(sProperty);
conn.connect();
InputStream input = conn.getInputStream();
//高速缓存,设置缓冲区
byte[] b = new byte[1024];
int nRead;
while((nRead=input.read(b,0,1024))>0&&this.nStartPos<this.nEndPos) {
System.out.println("================================="+this.nThreadID);
this.nStartPos += this.fileAccess.write(b, 0, nRead);
}
Utility.log("Thread "+nThreadID+" is over!");
this.bDownOver = true;
//一定要注意释放和connect,很关键
input.close();
conn.disconnect();
} catch(Exception e) {
e.printStackTrace();
}
}
}
//打印回应的头信息
public void logResponseHead(HttpURLConnection conn) {
for(int i=1; ; i++) {
String headerKey = conn.getHeaderFieldKey(i);
//key 不为空
if(headerKey!=null) {
Utility.log(headerKey+" : "+conn.getHeaderField(headerKey));
} else {
break;
}
}
}
public void splitterStop() {
this.bStop = true;
}
}
4.要抓取的文件的信息,如文件保存的目录,名字,抓取文件的 URL 等:
/**
* 要抓取的文件的信息,如文件保存的目录,名字,抓取文件的 URL 等。
* @author ETHAN
*
*/
public class SiteInfoBean {
private String sSiteURL;//site's url
private String sFilePath; //saved file's path
private String sFileName; //saved file's name
private int nSplitter; //count of splited downloading file
public SiteInfoBean() {
this("","","",5); //default value of nSplitter is 5
}
public SiteInfoBean(String sURL, String sPath, String sName, int nSplitter) {
// TODO Auto-generated constructor stub
this.sSiteURL = sURL;
this.sFileName = sName;
this.sFilePath = sPath;
this.nSplitter = nSplitter;
}
public String getsSiteURL() {
return sSiteURL;
}
public void setsSiteURL(String sSiteURL) {
this.sSiteURL = sSiteURL;
}
public String getsFilePath() {
return sFilePath;
}
public void setsFilePath(String sFilePath) {
this.sFilePath = sFilePath;
}
public String getsFileName() {
return sFileName;
}
public void setsFileName(String sFileName) {
this.sFileName = sFileName;
}
public int getnSplitter() {
return nSplitter;
}
public void setnSplitter(int nSplitter) {
this.nSplitter = nSplitter;
}
}
5.Utility.java
/*
**Utility.java
*/
public class Utility {
public Utility()
{
}
public static void sleep(int nSecond)
{
try{
Thread.sleep(nSecond);
}
catch(Exception e)
{
e.printStackTrace ();
}
}
public static void log(String sMsg)
{
System.err.println(sMsg);
}
public static void log(int sMsg)
{
System.err.println(sMsg);
}
}
经过测试,下载完全成功!中间有一些打印输出,没有注释掉。部分打印如下:
=================================2
=================================2
=================================2
=================================2
=================================2
=================================2
=================================2
Thread 2 is over!
---------ok------------------threadID-------------------0false
---------ok------------------threadID-------------------0true
---------ok------------------threadID-------------------1false
---------ok------------------threadID-------------------1true
---------ok------------------threadID-------------------2false
---------ok------------------threadID-------------------2true
---------ok------------------threadID-------------------3false
---------ok------------------threadID-------------------3true
---------ok------------------threadID-------------------4false
---------ok------------------threadID-------------------4true
break while---------------------------true
文件下载结束!