一个下载SVN源码+http协议的java实现

要求

1.不想引用多余jar

2.需要多线程下载

3.访问SVN要用户认证

 

实现代码

1.Main

package svnhttp;

public class Main {

 /**
  * @param args
  */
 public static void main(String[] args) {
  UrlElementBean bean = new UrlElementBean();
  String baseUrl = "";
  bean.setRelativeUrl("");
  InteractionBox.setBaseUrl(baseUrl);
  InteractionBox.pushMaterBean(bean);
  MainProductThread mainProduct = new MainProductThread();
  mainProduct.initThread();
  mainProduct.start();
  MainConsumeThread mainConsume = new MainConsumeThread();
  mainConsume.initThread();
  mainConsume.start();
 }

}

2.UrlElementBean

package svnhttp;

import java.util.ArrayList;
import java.util.List;

public class UrlElementBean {
 private List<String> files = new ArrayList<String>();
 private List<String> folds = new ArrayList<String>();
 private String relativeUrl;
 public List<String> getFiles() {
  return files;
 }
 public void setFiles(List<String> files) {
  this.files = files;
 }
 public List<String> getFolds() {
  return folds;
 }
 public void setFolds(List<String> folds) {
  this.folds = folds;
 }
 public String getRelativeUrl() {
  return relativeUrl;
 }
 public void setRelativeUrl(String relativeUrl) {
  this.relativeUrl = relativeUrl;
 }
 
}

3 InteractionBox

package svnhttp;

import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
import java.util.Queue;

public class InteractionBox {
 public static List<UrlElementBean> materBeanTmp = Collections.synchronizedList( new  LinkedList<UrlElementBean>());//不确定同步的作用
 public static List<UrlElementBean> productBeanTmp = Collections.synchronizedList(new LinkedList<UrlElementBean>());
 public static Queue<UrlElementBean> materBean = new  LinkedList<UrlElementBean>( materBeanTmp );
 public static Queue<UrlElementBean> productBean = new LinkedList<UrlElementBean>(productBeanTmp) ;
 public static String baseUrl;
 public static int MAX_MATER_NUMBER = 2000;
 public static int MAX_PRODUCT_NUMBER = 2000;
 
 public static String getBaseUrl() {
  return baseUrl;
 }

 public static void setBaseUrl(String bUrl) {
  baseUrl = bUrl;
 }

 

4.MainProductThread

package svnhttp;

import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
import java.util.Queue;

public class MainProductThread extends Thread {
// private Lock lock = new ReentrantLock();
 public static List<ProductThread> productThreadsTmp =   Collections.synchronizedList( new LinkedList<ProductThread>());
 public static Queue<ProductThread> productThreads =  new LinkedList<ProductThread>(productThreadsTmp);

 public static void pushProductThread( ProductThread pt ) {
  productThreads.add( pt );
 }
 
 public  void initThread(){
  for( int i=0;i<20;i++) {
   ProductThread pt = new ProductThread();
   productThreads.add(pt);
  }
 }
 
 public void run() {
  int count=5;
  while(true){
 
   ProductThread t= null;
//   lock.lock();
   if ( productThreads.size()>0){
    t = productThreads.poll();
    
    if (  InteractionBox.getMaterLineNumber()== 0 ){
     try {
      productThreads.add( t );
      sleep(1000);
      count = count -1;
     } catch (InterruptedException e) {
      e.printStackTrace();
     }
    } else if (InteractionBox.getMaterLineNumber()>0 ) {
     UrlElementBean urlElementBean = InteractionBox.getMaterBean();
     t.setUrlElementBean(urlElementBean);
     new Thread(t).start();
     count = 5;
    } else {
     productThreads.add( t );
     try {
      sleep(1000);
     } catch (InterruptedException e) {
      e.printStackTrace();
     } 
    }
   } else {
    try {
     sleep(1000);
    } catch (InterruptedException e) {
     e.printStackTrace();
    } 
   }
//   lock.unlock();
   if (count == 0 )
    break;
  }
 }
}

 
 public static UrlElementBean getMaterBean( ) {
  return materBean.poll();
 }
 
 public static UrlElementBean getProductBean() {
  return productBean.poll();
 }
 
 public static void pushMaterBean(UrlElementBean bean) {
  materBean.add(bean);
 }
 
 public static void pushProductBean(UrlElementBean bean) {
  productBean.add( bean );
 }
 
 public static int getMaterLineNumber(){
  return materBean.size();
 }
 
 public static int getProductLineNumber() {
  return productBean.size();
 }
 
}

5.MainConsumeThread

package svnhttp;

import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
import java.util.Queue;

public class MainConsumeThread extends Thread {
 public static List<ConsumeThread> consumeThreadsTmp =  Collections.synchronizedList(new LinkedList<ConsumeThread>());
 public static Queue<ConsumeThread> consumeThreads =   new LinkedList<ConsumeThread>(consumeThreadsTmp);
 
 public static void pushConsumeThread( ConsumeThread ct ) {
  consumeThreads.add( ct );
 }
 public  void initThread(){
  for( int i=0;i<20;i++) {
   ConsumeThread ct = new ConsumeThread();
   consumeThreads.add(ct);
  }
 }
 
 public void run() {
  int count=5;
  while(true) {
    ConsumeThread t=  null;
   if ( consumeThreads.size()>0){
    t = consumeThreads.poll();
    if (   InteractionBox.getProductLineNumber()== 0 && InteractionBox.getMaterLineNumber()==0 ){
     try {
      consumeThreads.add( t );
      sleep(1000);
      count = count -1;
     } catch (InterruptedException e) {
      e.printStackTrace();
     }
    } else if (InteractionBox.getProductLineNumber()>0 ) {
     UrlElementBean urlElementBean = InteractionBox.getProductBean();
     t.setUrlElementBean(urlElementBean);
     new Thread(t).start();
     count = 5;
    } else {
     consumeThreads.add( t );
     try {
      sleep(1000);
     } catch (InterruptedException e) {
      e.printStackTrace();
     } 
    }
   } else {
    try {
     sleep(1000);
    } catch (InterruptedException e) {
     e.printStackTrace();
    } 
   }
   if (count == 0 )
    break;
  }
 }
 
}

6.WebPageReader

package svnhttp;

import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.io.StringReader;
import java.net.Authenticator;
import java.net.URL;
import java.net.URLConnection;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;

import org.w3c.dom.Document;
import org.xml.sax.InputSource;

public class WebPageReader {

 private String baseUrl;
 
 public String getBaseUrl() {
  return baseUrl;
 }

 public void setBaseUrl(String baseUrl) {
  this.baseUrl = baseUrl;
 }

 public  Document getDocument() throws Exception {
  Document doc = null;
  //需要验证
   Authenticator.setDefault(new AuthImpl());
   URL url = new URL(baseUrl);
   URLConnection connection = url.openConnection();

   StringBuffer strBuf = new StringBuffer();
   String inputLine;
   BufferedReader in  = new BufferedReader(new InputStreamReader(connection.getInputStream()));
   while ((inputLine = in.readLine()) != null) {
    strBuf.append(inputLine);
   }
   String content = strBuf.toString();
   int beginIndex = content.indexOf("<ul>")+"<ul>".length();
   int endIndex =  content.lastIndexOf("</ul>");
   content ="<ul>"+ content.substring(beginIndex, endIndex)+"</ul>";
   DocumentBuilder docBuilder = DocumentBuilderFactory.newInstance().newDocumentBuilder() ;
   doc =docBuilder.parse( new  InputSource(new StringReader(content))  );
  return doc;
 }
 
}

7.AnalyzeDocument

package svnhttp;

import java.util.ArrayList;
import java.util.List;

import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NodeList;

public class AnalyzeDocument {
 
 private Document doc;
 private List<String> files = new ArrayList<String>();
 private List<String> folds = new ArrayList<String>();
 
 public void filterDocument(Document inputDoc,String relativeUrl ){
  this.doc = inputDoc;
  Element  ele =doc.getDocumentElement();
  NodeList ns =ele.getChildNodes();
  for(int i=0;i<ns.getLength();i++) {
   String nodeTxt = ns.item(i).getTextContent().trim();
   if (!nodeTxt.equals("")){
    if ( "..".equals( nodeTxt) ){
     
    }else if ( nodeTxt.substring(nodeTxt.length()-1, nodeTxt.length()).equals("/")){ // folder
     folds.add( nodeTxt );
    } else {
     files.add(nodeTxt);
    }
   }
  }
 }
 
 public List<String> getFolds() {
  return folds;
 }

 public void setFolds(List<String> folds) {
  this.folds = folds;
 }

 public List<String> getFiles() {
  return files;
 }

 public void setFiles(List<String> files) {
  this.files = files;
 }

}

8.AuthImpl

package svnhttp;

import java.net.Authenticator;
import java.net.PasswordAuthentication;

public class AuthImpl extends Authenticator {

 @Override
 protected PasswordAuthentication getPasswordAuthentication() {
  String user="";
  char[] pass="".toCharArray();
  return new PasswordAuthentication(user,pass);
 }

}

9.ConsumeThread

package svnhttp;

import java.io.File;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.Authenticator;
import java.net.URL;
import java.net.URLConnection;
import java.net.URLEncoder;
import java.util.List;

public class ConsumeThread  implements Runnable{
 
 private UrlElementBean urlElementBean;
 public UrlElementBean getUrlElementBean() {
  return urlElementBean;
 }
 public void setUrlElementBean(UrlElementBean urlElementBean) {
  this.urlElementBean = urlElementBean;
 }
 
 public void run() {
   try {
   Authenticator.setDefault(new AuthImpl());
   String relativeUrl = urlElementBean.getRelativeUrl();
   List<String> files = urlElementBean.getFiles();
   String baseoutput = "c:/tmp/";
   String outputPath = baseoutput;
   if (!relativeUrl.equals("")){
    String folderPath=outputPath+relativeUrl.substring(0,relativeUrl.length()-1);
    File f = new File(folderPath);
    if ( !f.exists()){
     f.mkdirs();
    }
   }
   
   outputPath = outputPath+relativeUrl;
   for(int i=0;i<files.size();i++){

//处理有空格的url地址
    String fileName = URLEncoder.encode(files.get(i).trim(),"UTF-8");
    fileName= fileName.replaceAll("//+","%20");
    String accessUrl = InteractionBox.getBaseUrl() + relativeUrl + fileName;
    URL url = new URL(  accessUrl);
    URLConnection connection = url.openConnection();
    InputStream in = connection.getInputStream();
    OutputStream out = new FileOutputStream(outputPath+files.get(i));
    int data = in.read();
    while( data !=-1){
     out.write(data);
     data = in.read();
    }
    in.close();
    out.close();
   }
  }catch(Exception e){
   e.printStackTrace();
  } finally {
   MainConsumeThread.pushConsumeThread( new ConsumeThread() );
  }
  
 }
 
}