java 之 断点续传和高速缓存

1.文件操作类，负责文件写入：

import java.io.IOException;
import java.io.RandomAccessFile;
import java.io.Serializable;


public class FileAccess implements Serializable  {
	private RandomAccessFile oSavedFile;
	private long nPos;
	
	public FileAccess() throws IOException {
		this("",0);
	}
	
	public FileAccess(String sName, long nPos) throws IOException {
		this.oSavedFile = new RandomAccessFile(sName, "rw");
		this.nPos = nPos;
		this.oSavedFile.seek(nPos);
	}
	
	//不需要同步，写得位置不一样
	public  int write(byte[] b, int nStart, int nLen) {
		int n = -1;
		try {
			this.oSavedFile.write(b,nStart,nLen);
			n = nLen;
		} catch(IOException e) {
			e.printStackTrace();
		}
		
		return n;
	}
}

2.文件下载器，负责启动 多线程下载：

import java.io.DataInputStream;  
import java.io.DataOutputStream;  
import java.io.File;  
import java.io.FileInputStream;  
import java.io.FileNotFoundException;
import java.io.FileOutputStream;  
import java.io.IOException;  
import java.net.HttpURLConnection;  
import java.net.URL;  
  
  
public class FileDownloader {  
    //文件信息Bean   
    private SiteInfoBean siteInfoBean = null;  
      
    //多个线程   
    private long[] nStartPos;//开始位置   
    private long[] nEndPos;//结束位置   
      
    //子线程对象   
    private FileSplitterFetch[] fileSplitterFetch;  
    //文件长度   
    private long nFileLength;   
    //是否第一次读取文件   
    private boolean bFirst = true;  
    //停止标志   
    //boolean bStop = false;   
    //文件下载的临时信息，保存下载信息   
    private File tmpFile;  
      
    //输出到文件的输出流   
    private DataOutputStream output;
    
    //下载完成的线程数量
    private int count;
    
    public FileDownloader() {
    	
    }  
    public FileDownloader(SiteInfoBean bean) {  
        // TODO Auto-generated constructor stub   
        this.siteInfoBean = bean;  
        this.tmpFile = new File(bean.getsFilePath()+File.separator+bean.getsFileName()+".info");  
        
      System.out.println(tmpFile.getAbsolutePath());
        if(this.tmpFile.exists()) {  
            this.bFirst = false;  
            //读取下载信息   
            read_pos();  
        } else {  
        	//splitter 文件被分割的份数
            this.nStartPos = new long[this.siteInfoBean.getnSplitter()];  
            this.nEndPos = new long[this.siteInfoBean.getnSplitter()];  
        }  
    }  
    public void read_pos() {  
        // TODO Auto-generated method stub   
        try {  
            DataInputStream input = new DataInputStream(new FileInputStream(tmpFile));  
            //第一个数字，线程数量
            int nCount = input.readInt();  
              
            this.nStartPos = new long[nCount];  
            this.nEndPos = new long[nCount];  
              
            for(int i=0;i<this.nStartPos.length;i++) {  
                this.nStartPos[i] = input.readLong();  
                this.nEndPos[i] = input.readLong();  
            }  
              
            input.close();  
        } catch(Exception e) {  
            e.printStackTrace();  
        }  
    }  
      
    public void download() throws IOException, InterruptedException {  
        //如果是第一次下载，没有info文件，则分配复制 起始 端点下载量   
        if(this.bFirst) {  
            //获取文件长度   
            this.nFileLength = getFileSize();  
            System.out.println("------------fileLen-----------------"+this.nFileLength);  
            if(this.nFileLength==-1) {  
                System.err.println("File length is not know");  
            } else if(this.nFileLength==-2){  
                System.err.println("File is not access!");  
            } else {  
                  
                //block开头   
                for(int i=0;i<this.nStartPos.length;i++) {  
                    //nStartPos.length是自己规定的   
                    this.nStartPos[i] = (long)(i*(this.nFileLength/this.nStartPos.length));  
                }  
                //block末尾   
                for(int i=0;i<this.nEndPos.length-1;i++) {  
                    this.nEndPos[i] = this.nStartPos[i+1];  
                }  
                this.nEndPos[this.nEndPos.length-1] = this.nFileLength;  
              
                //分配好之后，创建断点记录文件
                try {
        			this.output = new DataOutputStream(new FileOutputStream(tmpFile));
        		} catch (FileNotFoundException e) {
        			e.printStackTrace();
        		}  
            }  
        }  
          
        //启动子线程   
          
        this.fileSplitterFetch = new FileSplitterFetch[this.nStartPos.length];  
        System.out.println("---------------------------------");  
        for(int i=0;i<this.nStartPos.length;i++) {  
            this.fileSplitterFetch[i] = new FileSplitterFetch(this.siteInfoBean.getsSiteURL(),  
                    this.siteInfoBean.getsFilePath()+File.separator+this.siteInfoBean.getsFileName(),  
                    this.nStartPos[i],this.nEndPos[i],i);  
              
            System.out.println("Thread "+i+" , nStartPos= "+this.nStartPos[i]+" , nEndPos="+this.nEndPos[i]);  
            this.fileSplitterFetch[i].start();  
        }  
          
          
        //是否结束while循环   
        boolean breakWhile = false;  
        while(!breakWhile) {  
            //   
            System.out.println("开始while。。。。。。。。");  
            //write_nPos();   
            //Utility.sleep(1000);   
            //Thread.sleep(1000); 
          //断点 记录
            write_nPos();
          //必须sleep，否则记录文件会被清空，没有数据 0kb
            Utility.sleep(500);
            
            breakWhile = true;  
   /*---------------------+++++++++++++++++++++++++++*/
            for(int i=0;i<this.nStartPos.length;i++) {  
                //System.out.println("i++++++++++++++++++++++++++++++++++"+i);   
                //还没下载完成，检查如果有一个没有下载完就设置 breakWhile=false   
                if(!this.fileSplitterFetch[i].bDownOver) {  
                    breakWhile = false;  
                    
                    break;  
                //单个线程 下载完成   
                } else if(!this.fileSplitterFetch[i].bStop) {  
                    System.out.println("---------ok------------------"+"threadID-------------------"+i+this.fileSplitterFetch[i].bStop);  
                    //停止线程   
                    this.fileSplitterFetch[i].splitterStop();    
                    System.out.println("---------ok------------------"+"threadID-------------------"+i+this.fileSplitterFetch[i].bStop);  
                    count++; 
                    System.out.println("-------------------------count-----------------"+count); 
                    //如果5个线程都下载完成，停止while.   
                    //有bug，如果接着上次的下载，count又重新计数了，没有意义了   
                    if(count>=this.fileSplitterFetch.length) breakWhile = true;   
                }  
            }  
            System.out.println("break while---------------------------"+breakWhile);  
            //breakWhile=false如果没有下载完成，各个线程   
            if(breakWhile) {  
                System.out.println("文件下载结束！");  
                for(int i=0;i<nStartPos.length;i++)   
                     fileSplitterFetch[i].splitterStop();   
                break;  
            }  
            //Thread.sleep(1000);  
        }  
    }  
    
    //保存下载信息（文件指针位置)   
        private  void write_nPos() {  
            // TODO Auto-generated method stub   
            try {
            	
        		this.output = new DataOutputStream(new FileOutputStream(tmpFile));
        		
                //下载线程数量   
                this.output.writeInt(this.nStartPos.length);  
                  
                for(int i=0;i<this.nStartPos.length;i++) {  
                    this.output.writeLong(this.fileSplitterFetch[i].nStartPos);  
                    this.output.writeLong(this.fileSplitterFetch[i].nEndPos);  
                }  
                this.output.flush();
                this.output.close();  
            } catch(Exception e) {  
                e.printStackTrace();  
            }  
        }  
          
    //获得文件长度   
        private long getFileSize() {  
            // TODO Auto-generated method stub   
            int nFileLength = -1;  
              
            try {  
                URL url = new URL(this.siteInfoBean.getsSiteURL());  
                HttpURLConnection conn = (HttpURLConnection) url.openConnection();  
                conn.setRequestProperty("User-Agent",    
                        "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.3) Gecko/2008092510 Ubuntu/8.04 (hardy) Firefox/3.0.3");    
                  
                int responseCode = conn.getResponseCode();  
                  
                if(responseCode>=400) {  
                    processErrorCode(responseCode);  
                    return -2; //-2 represents access is error   
                }  
                /*String sHeader; 
                for(int i=1; ; i++) { 
                    sHeader = conn.getHeaderFieldKey(i); 
                    if(sHeader!=null) { 
                        if(sHeader.equals("Content-Length")) { 
                            nFileLength = Integer.parseInt(conn.getHeaderField(sHeader)); 
                            break; 
                        } 
                    } else { 
                        break; 
                    } 
                }*/  
                  
                nFileLength = conn.getContentLength();  
            } catch(Exception e) {  
                e.printStackTrace();  
            }  
            return nFileLength;  
        }  
        private void processErrorCode(int responseCode) {  
            // TODO Auto-generated method stub   
            System.err.println("Error Code: "+responseCode);  
        }  
          
        public static void main(String args[]) throws IOException, InterruptedException {  
            SiteInfoBean bean = new SiteInfoBean("http://127.0.0.1:8080/fileTest/download/a.zip","d:/test","a.zip",1);  
            //SiteInfoBean bean1 = new SiteInfoBean("http://localhost/download/friends.rmvb","C:\\xampp\\htdocs\\test","hello.rmvb",5);  
            FileDownloader downloader = new FileDownloader(bean);  
            downloader.download();  
        }  
}  

3. 负责部分文件的抓取的线程：

import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URL;

/**
 * 负责部分文件的抓取, 多线程
 * @author ETHAN
 *
 */
public class FileSplitterFetch extends Thread {
	private String sURL; //file url
    long nStartPos;//file snippet start position
    long nEndPos; //file snippet end position
	
	private int nThreadID;
	boolean bDownOver = false;//downing is over;
	boolean bStop = false; //stop identical相同
	
	private FileAccess fileAccess = null; // file access interface
	
	public FileSplitterFetch(String sURL,String sName,long nStart,long nEnd,int id) throws IOException {
		this.sURL = sURL;
		this.nStartPos = nStart;
		this.nEndPos = nEnd;
		this.nThreadID = id;
		this.fileAccess = new FileAccess(sName,this.nStartPos);
	}
	
	public void run() {
		while(this.nStartPos<this.nEndPos&&!this.bStop) {
			System.out.println("-----------------------------------"+this.nThreadID+"---------开始写数据");
			try {
				URL url = new URL(sURL);
				HttpURLConnection conn = (HttpURLConnection) url.openConnection();
				conn.setRequestProperty("User-Agent",  
                        "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.3) Gecko/2008092510 Ubuntu/8.04 (hardy) Firefox/3.0.3");  
				
				String sProperty = "bytes="+this.nStartPos+"-";
				conn.setRequestProperty("range", sProperty);
				
				Utility.log(sProperty);
				conn.connect();
				InputStream input = conn.getInputStream();
				
				//高速缓存，设置缓冲区
				byte[] b = new byte[1024];
				int nRead;
				
				while((nRead=input.read(b,0,1024))>0&&this.nStartPos<this.nEndPos) {
					System.out.println("================================="+this.nThreadID);
					this.nStartPos += this.fileAccess.write(b, 0, nRead);
				}
				
				Utility.log("Thread "+nThreadID+" is over!");
				this.bDownOver = true;
				
				//一定要注意释放和connect，很关键
				input.close();
				conn.disconnect();
			} catch(Exception e) {
				e.printStackTrace();
			}
		}
	}
	
	//打印回应的头信息
	public void logResponseHead(HttpURLConnection conn) {
		for(int i=1; ; i++) {
			String headerKey = conn.getHeaderFieldKey(i);
			
			//key 不为空
			if(headerKey!=null) {
				Utility.log(headerKey+" : "+conn.getHeaderField(headerKey));
			} else {
				break;
			}
		}
	}
	
	public void splitterStop() {
		this.bStop = true;
	}
}

4.要抓取的文件的信息，如文件保存的目录，名字，抓取文件的 URL 等：

/**
 *  要抓取的文件的信息，如文件保存的目录，名字，抓取文件的 URL 等。 
 * @author ETHAN
 *
 */
public class SiteInfoBean {
	private String sSiteURL;//site's url
	private String sFilePath; //saved file's path
	private String sFileName; //saved file's name
	private int nSplitter; //count of splited downloading file
	
	public SiteInfoBean() {
		this("","","",5); //default value of nSplitter is 5
	}

	public SiteInfoBean(String sURL, String sPath, String sName, int nSplitter) {
		// TODO Auto-generated constructor stub
		this.sSiteURL = sURL;
		this.sFileName = sName;
		this.sFilePath = sPath;
		this.nSplitter = nSplitter;
	}

	public String getsSiteURL() {
		return sSiteURL;
	}

	public void setsSiteURL(String sSiteURL) {
		this.sSiteURL = sSiteURL;
	}

	public String getsFilePath() {
		return sFilePath;
	}

	public void setsFilePath(String sFilePath) {
		this.sFilePath = sFilePath;
	}

	public String getsFileName() {
		return sFileName;
	}

	public void setsFileName(String sFileName) {
		this.sFileName = sFileName;
	}

	public int getnSplitter() {
		return nSplitter;
	}

	public void setnSplitter(int nSplitter) {
		this.nSplitter = nSplitter;
	}
}

5.Utility.java

/* 
 **Utility.java 
 */ 
 public class Utility { 
 public Utility() 
 { 
 } 
 public static void sleep(int nSecond) 
 { 
 try{ 
 Thread.sleep(nSecond); 
 } 
 catch(Exception e) 
 { 
 e.printStackTrace (); 
 } 
 } 
 public static void log(String sMsg) 
 { 
 System.err.println(sMsg); 
 } 
 public static void log(int sMsg) 
 { 
 System.err.println(sMsg); 
 } 
 } 

 

经过测试，下载完全成功！中间有一些打印输出，没有注释掉。部分打印如下：

=================================2
=================================2
=================================2
=================================2
=================================2
=================================2
=================================2
Thread 2 is over!
---------ok------------------threadID-------------------0false
---------ok------------------threadID-------------------0true
---------ok------------------threadID-------------------1false
---------ok------------------threadID-------------------1true
---------ok------------------threadID-------------------2false
---------ok------------------threadID-------------------2true
---------ok------------------threadID-------------------3false
---------ok------------------threadID-------------------3true
---------ok------------------threadID-------------------4false
---------ok------------------threadID-------------------4true
break while---------------------------true
文件下载结束！