java多线程控制下载_java多线程下载控制器

package com.spider.crawler;

import java.util.ArrayList;

import java.util.Iterator;

import com.spider.link.db.Linkdb;

import com.spider.log.Log;

import com.spider.util.Link;

import com.spider.util.MD5;

public class Controller {

private int thread_num = 0;

private boolean flag = true;

private Downloder downloder = null;

//用一个ArrayList作为线程池，增删起来比较方便

private ArrayList downloaderPool = null;

//同上

private Log mLog = null;

//这个不是线程控制器需要的类

private Linkdb linkdb = null;

public Controller() {

mLog = Log.getInstance();

linkdb = Linkdb.getLinkdb();

}

public void initThreadPoolSize() {

this.thread_num = 5;

createMulThread();

}

public void initThreadPoolSize(int num) {

this.thread_num = num;

createMulThread();

}

private void createMulThread() {

mLog.runstat("create " + this.thread_num + " thread");

downloaderPool = new ArrayList();

for (int i = 0; i < thread_num; i++) {

downloder = new Downloder("Thread-" + i);

downloaderPool.add(downloder);

}

//启动线程池中所有线程

startThreadPool();

}

private void startThreadPool() {

mLog.runstat("start all thread in pool");

for (Iterator iter = downloaderPool.iterator(); iter

.hasNext();) {

iter.next().start();

}

}

public void initUnvisitedQueue(String seeds[]) {

for (int i = 0; i < seeds.length; i++) {

Link link = new Link();

link.setUrlKey(MD5.getMD5Code(seeds[i]));

link.setUrlValue(seeds[i]);

linkdb.addUnvisitedTb(link);

}

}

// 调度线程执行

public void schedure() {

new Thread() {

public void run() {

while (flag) {

// 还有未检索的链接，并且还有空闲线程，则分配任务

if (!checkUnVisitedQueueEmpty()) {

Downloder downloder = hasRunnableThread();

if(null != downloder){

// 线程分配任务：从线程池中取出一个线程，将其状态置为RUNNING

// 取池中 第一条线程，并唤醒线程开始执行

System.out.println("----"+downloder.getName()+"----");

downloder.setThreadState(downloder.RUNNING);

downloder.notifyThread();

}else{

continue;

}

}else if (checkQuit()) {

// 所有线程均已执行完毕，并且未访问队列为空,则退出线程调度

System.out.println("quit");

quit();

}

}

}

}.start();

}

private  boolean checkQuit() {

for (int i=0;i

Downloder dr = downloaderPool.get(i);

if (dr.getThreadState() == dr.RUNNING) {

return false;

}

}

return true;

}

private boolean checkUnVisitedQueueEmpty() {

// 如果还有未检索的URL的继续检索 fg = true;

if (linkdb.unvisitedEmpty()) {

return true;

} else {

// 否则返回false fg = true;

return false;

}

}

private synchronized Downloder hasRunnableThread() {

for(Iterator

iter=downloaderPool.iterator();iter.hasNext();){

Downloder dr = iter.next();

if(dr.getThreadState() == dr.PAUSED){

return dr;

}

}

return null;

}

private synchronized void quit() {

// 回收线程池资源,退出多线程

while(downloaderPool.size() > 0){

Downloder dloader = downloaderPool.remove(0);

dloader.setFlag(false);

dloader.notify();

}

this.flag = false;

}

}