要求
1.不想引用多余jar
2.需要多线程下载
3.访问SVN要用户认证
实现代码
1.Main
package svnhttp;
public class Main {
/**
* @param args
*/
public static void main(String[] args) {
UrlElementBean bean = new UrlElementBean();
String baseUrl = "";
bean.setRelativeUrl("");
InteractionBox.setBaseUrl(baseUrl);
InteractionBox.pushMaterBean(bean);
MainProductThread mainProduct = new MainProductThread();
mainProduct.initThread();
mainProduct.start();
MainConsumeThread mainConsume = new MainConsumeThread();
mainConsume.initThread();
mainConsume.start();
}
}
2.UrlElementBean
package svnhttp;
import java.util.ArrayList;
import java.util.List;
public class UrlElementBean {
private List<String> files = new ArrayList<String>();
private List<String> folds = new ArrayList<String>();
private String relativeUrl;
public List<String> getFiles() {
return files;
}
public void setFiles(List<String> files) {
this.files = files;
}
public List<String> getFolds() {
return folds;
}
public void setFolds(List<String> folds) {
this.folds = folds;
}
public String getRelativeUrl() {
return relativeUrl;
}
public void setRelativeUrl(String relativeUrl) {
this.relativeUrl = relativeUrl;
}
}
3 InteractionBox
package svnhttp;
import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
import java.util.Queue;
public class InteractionBox {
public static List<UrlElementBean> materBeanTmp = Collections.synchronizedList( new LinkedList<UrlElementBean>());//不确定同步的作用
public static List<UrlElementBean> productBeanTmp = Collections.synchronizedList(new LinkedList<UrlElementBean>());
public static Queue<UrlElementBean> materBean = new LinkedList<UrlElementBean>( materBeanTmp );
public static Queue<UrlElementBean> productBean = new LinkedList<UrlElementBean>(productBeanTmp) ;
public static String baseUrl;
public static int MAX_MATER_NUMBER = 2000;
public static int MAX_PRODUCT_NUMBER = 2000;
public static String getBaseUrl() {
return baseUrl;
}
public static void setBaseUrl(String bUrl) {
baseUrl = bUrl;
}
4.MainProductThread
package svnhttp;
import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
import java.util.Queue;
public class MainProductThread extends Thread {
// private Lock lock = new ReentrantLock();
public static List<ProductThread> productThreadsTmp = Collections.synchronizedList( new LinkedList<ProductThread>());
public static Queue<ProductThread> productThreads = new LinkedList<ProductThread>(productThreadsTmp);
public static void pushProductThread( ProductThread pt ) {
productThreads.add( pt );
}
public void initThread(){
for( int i=0;i<20;i++) {
ProductThread pt = new ProductThread();
productThreads.add(pt);
}
}
public void run() {
int count=5;
while(true){
ProductThread t= null;
// lock.lock();
if ( productThreads.size()>0){
t = productThreads.poll();
if ( InteractionBox.getMaterLineNumber()== 0 ){
try {
productThreads.add( t );
sleep(1000);
count = count -1;
} catch (InterruptedException e) {
e.printStackTrace();
}
} else if (InteractionBox.getMaterLineNumber()>0 ) {
UrlElementBean urlElementBean = InteractionBox.getMaterBean();
t.setUrlElementBean(urlElementBean);
new Thread(t).start();
count = 5;
} else {
productThreads.add( t );
try {
sleep(1000);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
} else {
try {
sleep(1000);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
// lock.unlock();
if (count == 0 )
break;
}
}
}
public static UrlElementBean getMaterBean( ) {
return materBean.poll();
}
public static UrlElementBean getProductBean() {
return productBean.poll();
}
public static void pushMaterBean(UrlElementBean bean) {
materBean.add(bean);
}
public static void pushProductBean(UrlElementBean bean) {
productBean.add( bean );
}
public static int getMaterLineNumber(){
return materBean.size();
}
public static int getProductLineNumber() {
return productBean.size();
}
}
5.MainConsumeThread
package svnhttp;
import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
import java.util.Queue;
public class MainConsumeThread extends Thread {
public static List<ConsumeThread> consumeThreadsTmp = Collections.synchronizedList(new LinkedList<ConsumeThread>());
public static Queue<ConsumeThread> consumeThreads = new LinkedList<ConsumeThread>(consumeThreadsTmp);
public static void pushConsumeThread( ConsumeThread ct ) {
consumeThreads.add( ct );
}
public void initThread(){
for( int i=0;i<20;i++) {
ConsumeThread ct = new ConsumeThread();
consumeThreads.add(ct);
}
}
public void run() {
int count=5;
while(true) {
ConsumeThread t= null;
if ( consumeThreads.size()>0){
t = consumeThreads.poll();
if ( InteractionBox.getProductLineNumber()== 0 && InteractionBox.getMaterLineNumber()==0 ){
try {
consumeThreads.add( t );
sleep(1000);
count = count -1;
} catch (InterruptedException e) {
e.printStackTrace();
}
} else if (InteractionBox.getProductLineNumber()>0 ) {
UrlElementBean urlElementBean = InteractionBox.getProductBean();
t.setUrlElementBean(urlElementBean);
new Thread(t).start();
count = 5;
} else {
consumeThreads.add( t );
try {
sleep(1000);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
} else {
try {
sleep(1000);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
if (count == 0 )
break;
}
}
}
6.WebPageReader
package svnhttp;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.io.StringReader;
import java.net.Authenticator;
import java.net.URL;
import java.net.URLConnection;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import org.w3c.dom.Document;
import org.xml.sax.InputSource;
public class WebPageReader {
private String baseUrl;
public String getBaseUrl() {
return baseUrl;
}
public void setBaseUrl(String baseUrl) {
this.baseUrl = baseUrl;
}
public Document getDocument() throws Exception {
Document doc = null;
//需要验证
Authenticator.setDefault(new AuthImpl());
URL url = new URL(baseUrl);
URLConnection connection = url.openConnection();
StringBuffer strBuf = new StringBuffer();
String inputLine;
BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream()));
while ((inputLine = in.readLine()) != null) {
strBuf.append(inputLine);
}
String content = strBuf.toString();
int beginIndex = content.indexOf("<ul>")+"<ul>".length();
int endIndex = content.lastIndexOf("</ul>");
content ="<ul>"+ content.substring(beginIndex, endIndex)+"</ul>";
DocumentBuilder docBuilder = DocumentBuilderFactory.newInstance().newDocumentBuilder() ;
doc =docBuilder.parse( new InputSource(new StringReader(content)) );
return doc;
}
}
7.AnalyzeDocument
package svnhttp;
import java.util.ArrayList;
import java.util.List;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NodeList;
public class AnalyzeDocument {
private Document doc;
private List<String> files = new ArrayList<String>();
private List<String> folds = new ArrayList<String>();
public void filterDocument(Document inputDoc,String relativeUrl ){
this.doc = inputDoc;
Element ele =doc.getDocumentElement();
NodeList ns =ele.getChildNodes();
for(int i=0;i<ns.getLength();i++) {
String nodeTxt = ns.item(i).getTextContent().trim();
if (!nodeTxt.equals("")){
if ( "..".equals( nodeTxt) ){
}else if ( nodeTxt.substring(nodeTxt.length()-1, nodeTxt.length()).equals("/")){ // folder
folds.add( nodeTxt );
} else {
files.add(nodeTxt);
}
}
}
}
public List<String> getFolds() {
return folds;
}
public void setFolds(List<String> folds) {
this.folds = folds;
}
public List<String> getFiles() {
return files;
}
public void setFiles(List<String> files) {
this.files = files;
}
}
8.AuthImpl
package svnhttp;
import java.net.Authenticator;
import java.net.PasswordAuthentication;
public class AuthImpl extends Authenticator {
@Override
protected PasswordAuthentication getPasswordAuthentication() {
String user="";
char[] pass="".toCharArray();
return new PasswordAuthentication(user,pass);
}
}
9.ConsumeThread
package svnhttp;
import java.io.File;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.Authenticator;
import java.net.URL;
import java.net.URLConnection;
import java.net.URLEncoder;
import java.util.List;
public class ConsumeThread implements Runnable{
private UrlElementBean urlElementBean;
public UrlElementBean getUrlElementBean() {
return urlElementBean;
}
public void setUrlElementBean(UrlElementBean urlElementBean) {
this.urlElementBean = urlElementBean;
}
public void run() {
try {
Authenticator.setDefault(new AuthImpl());
String relativeUrl = urlElementBean.getRelativeUrl();
List<String> files = urlElementBean.getFiles();
String baseoutput = "c:/tmp/";
String outputPath = baseoutput;
if (!relativeUrl.equals("")){
String folderPath=outputPath+relativeUrl.substring(0,relativeUrl.length()-1);
File f = new File(folderPath);
if ( !f.exists()){
f.mkdirs();
}
}
outputPath = outputPath+relativeUrl;
for(int i=0;i<files.size();i++){
//处理有空格的url地址
String fileName = URLEncoder.encode(files.get(i).trim(),"UTF-8");
fileName= fileName.replaceAll("//+","%20");
String accessUrl = InteractionBox.getBaseUrl() + relativeUrl + fileName;
URL url = new URL( accessUrl);
URLConnection connection = url.openConnection();
InputStream in = connection.getInputStream();
OutputStream out = new FileOutputStream(outputPath+files.get(i));
int data = in.read();
while( data !=-1){
out.write(data);
data = in.read();
}
in.close();
out.close();
}
}catch(Exception e){
e.printStackTrace();
} finally {
MainConsumeThread.pushConsumeThread( new ConsumeThread() );
}
}
}