package com.spider.crawler;
import java.util.ArrayList;
import java.util.Iterator;
import com.spider.link.db.Linkdb;
import com.spider.log.Log;
import com.spider.util.Link;
import com.spider.util.MD5;
public class Controller {
private int thread_num = 0;
private boolean flag = true;
private Downloder downloder = null;
//用一个ArrayList作为线程池,增删起来比较方便
private ArrayList downloaderPool = null;
//同上
private Log mLog = null;
//这个不是线程控制器需要的类
private Linkdb linkdb = null;
public Controller() {
mLog = Log.getInstance();
linkdb = Linkdb.getLinkdb();
}
public void initThreadPoolSize() {
this.thread_num = 5;
createMulThread();
}
public void initThreadPoolSize(int num) {
this.thread_num = num;
createMulThread();
}
private void createMulThread() {
mLog.runstat("create " + this.thread_num + " thread");
downloaderPool = new ArrayList();
for (int i = 0; i < thread_num; i++) {
downloder = new Downloder("Thread-" + i);
downloaderPool.add(downloder);
}
//启动线程池中所有线程
startThreadPool();
}
private void startThreadPool() {
mLog.runstat("start all thread in pool");
for (Iterator iter = downloaderPool.iterator(); iter
.hasNext();) {
iter.next().start();
}
}
public void initUnvisitedQueue(String seeds[]) {
for (int i = 0; i < seeds.length; i++) {
Link link = new Link();
link.setUrlKey(MD5.getMD5Code(seeds[i]));
link.setUrlValue(seeds[i]);
linkdb.addUnvisitedTb(link);
}
}
// 调度线程执行
public void schedure() {
new Thread() {
public void run() {
while (flag) {
// 还有未检索的链接,并且还有空闲线程,则分配任务
if (!checkUnVisitedQueueEmpty()) {
Downloder downloder = hasRunnableThread();
if(null != downloder){
// 线程分配任务:从线程池中取出一个线程,将其状态置为RUNNING
// 取池中 第一条线程,并唤醒线程开始执行
System.out.println("----"+downloder.getName()+"----");
downloder.setThreadState(downloder.RUNNING);
downloder.notifyThread();
}else{
continue;
}
}else if (checkQuit()) {
// 所有线程均已执行完毕,并且未访问队列为空,则退出线程调度
System.out.println("quit");
quit();
}
}
}
}.start();
}
private boolean checkQuit() {
for (int i=0;i
Downloder dr = downloaderPool.get(i);
if (dr.getThreadState() == dr.RUNNING) {
return false;
}
}
return true;
}
private boolean checkUnVisitedQueueEmpty() {
// 如果还有未检索的URL的继续检索 fg = true;
if (linkdb.unvisitedEmpty()) {
return true;
} else {
// 否则返回false fg = true;
return false;
}
}
private synchronized Downloder hasRunnableThread() {
for(Iterator
iter=downloaderPool.iterator();iter.hasNext();){
Downloder dr = iter.next();
if(dr.getThreadState() == dr.PAUSED){
return dr;
}
}
return null;
}
private synchronized void quit() {
// 回收线程池资源,退出多线程
while(downloaderPool.size() > 0){
Downloder dloader = downloaderPool.remove(0);
dloader.setFlag(false);
dloader.notify();
}
this.flag = false;
}
}