1、文件扫描任务线程
/**
*
*/
package com.godway.filescanner;
import java.io.File;
import java.util.Date;
/**
* 递归调用
* 文件夹的文件扫描
*
* @author 高伟
* @date Jan 9, 2012 11:16:39 AM
* @description:
*/
public class DirectoryScanner implements Runnable {
private File dir;
private Date createTime;//创建时间
private Date startTime;//开始时间
private Date endTime;//结束时间
private boolean isSuccess;//是否成功
private IScannerVisitor visitor;
public DirectoryScanner(File dir) {
this.dir = dir;
this.createTime = new Date();
this.startTime = new Date();
this.endTime = new Date();
this.isSuccess = false;
}
public DirectoryScanner(File dir, IScannerVisitor visitor) {
this(dir);
this.visitor = visitor;
}
@Override
public void run() {
this.startTime = new Date();
try {
if(dir.isDirectory()){
System.out.println("start_dire:" + dir.getPath());
File[] fileList = dir.listFiles();
for (int i = 0; i < fileList.length; i++) {
File file = fileList[i];
DirectoryScanner sub = new DirectoryScanner(file,visitor);
Thread subThread = new Thread(sub);
subThread.start();
try {
/**
* join关键字:
* 子线程增加到主线程中,
* 保证所有子线程结束后,才结束主线程
*/
subThread.join();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
System.out.println("end_dire:" + dir.getPath());
}else{
System.out.println("file:" + dir.getPath());
if(visitor != null){
visitor.visitor(dir);
}
}
this.isSuccess = true;
} catch (Throwable e) {
this.isSuccess = false;
e.printStackTrace();
}
this.endTime = new Date();
}
//省略get set method
}
2、文件处理任务调度线程
package com.godway.filescanner;
import java.io.File;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
/**
* 负责分发文件
* @author 高伟
* @date Jan 9, 2012 6:28:15 PM
* @description:
*/
public class FileProcessor implements Runnable {
private IScannerVisitor visitor;
private ExecutorService pools;
public FileProcessor(IScannerVisitor visitor) {
this.visitor = visitor;
/**
* 使用无大小限制的连接池
* 通过下边任务控制,完成线程数量的控制
*/
this.pools = Executors.newCachedThreadPool();
}
@Override
public void run() {
while(true){
List<File> fileList = (List<File>) visitor.getScannerResult();
synchronized (fileList) {
if (fileList.size() == 0) {
try {
/**
* 处理完,没有待处理的,等待
*/
fileList.wait();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
/**
* 根据要处理的任务数,控制线程数量,避免过多性能上的消耗
* 避免高峰请求等因素
*/
if(fileList.size() <= 100){
scheduleTasks(20);
}else if(fileList.size()>100){
scheduleTasks(80);
}else if(fileList.size()>1000){
scheduleTasks(100);
}else if(fileList.size()>2000){
scheduleTasks(500);
}
}
}
private void scheduleTasks(int count){
for (int j = 0; j < count; j++) {
pools.submit(new FileProceTask((List<File>)visitor.getScannerResult()));
}
}
}
3、文件处理任务单元
package com.godway.filescanner;
import java.io.File;
import java.util.List;
/**
* 单个文件处理单元
* @author 高伟
* @date Jan 9, 2012 6:28:31 PM
* @description:
*/
public class FileProceTask implements Runnable {
private List<File> fileList;
public FileProceTask(List<File> fileList) {
this.fileList = fileList;
}
@Override
public void run() {
synchronized(fileList){
if(fileList.size() == 0){
try {
fileList.wait();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
File file = fileList.remove(fileList.size() - 1);
System.out.println("process file : " + file.getPath());
/**
* db operation etc.
* ...
*/
}
}
4、扫描器访问者接口
/**
*
*/
package com.godway.filescanner;
/**
* 扫描器访问者接口
* @author 高伟
* @date Jan 9, 2012 5:43:39 PM
* @description:
*/
public interface IScannerVisitor {
/**
*
* visitor
* @param obj
* 高伟
* Jan 9, 2012 7:13:59 PM
*/
public void visitor(Object obj);
/**
* 返回扫描结果
* getScannerResult
* @return
* 高伟
* Jan 9, 2012 7:13:35 PM
*/
public Object getScannerResult();
}
5、扫描器访问者实现之文件访问者
/**
*
*/
package com.godway.filescanner;
import java.io.File;
import java.util.LinkedList;
import java.util.List;
/**
* 扫描器之访问者实现--文件访问者
* @author 高伟
* @date Jan 9, 2012 5:43:12 PM
* @description:
*/
public class FileScannerVisitor implements IScannerVisitor {
private List<File> fileList = new LinkedList<File>();
/*
* (non-Javadoc)
* @see com.godway.filescanner.IScannerVisitor#visitor(java.lang.Object)
*/
@Override
public void visitor(Object obj) {
if(obj instanceof File){
File file = (File)obj;
synchronized(fileList){
if(!fileList.contains(file)){
fileList.add(file);
}
fileList.notify();
}
}
}
/*
* (non-Javadoc)
* @see com.godway.filescanner.IScannerVisitor#getScannerResult()
*/
@Override
public Object getScannerResult() {
return fileList;
}
//省略 get set method
}
6、扫描任务测试类
/**
*
*/
package com.godway.filescanner;
import java.io.File;
/**
* @author 高伟
* @date Jan 9, 2012 5:22:32 PM
* @description:
*/
public class DirectoryScannerTest {
public static void main(String[] args) {
File baseDir = new File("E:/work/");
FileScannerVisitor visitor = new FileScannerVisitor();
/**
* 启动扫描线程
*/
DirectoryScanner scanner = new DirectoryScanner(baseDir, visitor);
Thread thread = new Thread(scanner);
thread.start();
/**
* 等待扫描完成
*/
try {
thread.join();
} catch (InterruptedException e) {
e.printStackTrace();
}
/**
* 打印扫描结果
*/
System.out.println();
System.out.println("=-=-=-=-=-=-=-=-=结果-=-=-=-=-=-=-=-=-=");
System.out.println("Directory:" + scanner.getDir().getPath());
System.out.println("CreateTime:"+scanner.getCreateTime());
System.out.println("StartTime:"+scanner.getStartTime());
System.out.println("EndTime:"+scanner.getEndTime());
System.out.println("FileNum:"+visitor.getFileList().size());
System.out.println("=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=");
}
}
7、扫描加处理综合测试类
package com.godway.filescanner;
import java.io.File;
public class FileProcessorTest {
public static void main(String[] args) {
File baseDir = new File("E:/work");
FileScannerVisitor visitor = new FileScannerVisitor();
/**
* 启动处理线程
*/
FileProcessor processor = new FileProcessor(visitor);
Thread procThread = new Thread(processor);
procThread.start();
/**
* 启动扫描线程
*/
DirectoryScanner scanner = new DirectoryScanner(baseDir, visitor);
Thread scanThread = new Thread(scanner);
scanThread.start();
/**
* 等待扫描完成
*/
try {
scanThread.join();
} catch (InterruptedException e) {
e.printStackTrace();
}
/**
* 打印扫描结果
*/
System.out.println();
System.out.println("=-=-=-=-=-=-=-=-=结果-=-=-=-=-=-=-=-=-=");
System.out.println("Directory:" + scanner.getDir().getPath());
System.out.println("CreateTime:"+scanner.getCreateTime());
System.out.println("StartTime:"+scanner.getStartTime());
System.out.println("EndTime:"+scanner.getEndTime());
}
}
8、源码打包下载地址: