1、文件扫描任务线程
/** * */ package com.godway.filescanner; import java.io.File; import java.util.Date; /** * 递归调用 * 文件夹的文件扫描 * * @author 高伟 * @date Jan 9, 2012 11:16:39 AM * @description: */ public class DirectoryScanner implements Runnable { private File dir; private Date createTime;//创建时间 private Date startTime;//开始时间 private Date endTime;//结束时间 private boolean isSuccess;//是否成功 private IScannerVisitor visitor; public DirectoryScanner(File dir) { this.dir = dir; this.createTime = new Date(); this.startTime = new Date(); this.endTime = new Date(); this.isSuccess = false; } public DirectoryScanner(File dir, IScannerVisitor visitor) { this(dir); this.visitor = visitor; } @Override public void run() { this.startTime = new Date(); try { if(dir.isDirectory()){ System.out.println("start_dire:" + dir.getPath()); File[] fileList = dir.listFiles(); for (int i = 0; i < fileList.length; i++) { File file = fileList[i]; DirectoryScanner sub = new DirectoryScanner(file,visitor); Thread subThread = new Thread(sub); subThread.start(); try { /** * join关键字: * 子线程增加到主线程中, * 保证所有子线程结束后,才结束主线程 */ subThread.join(); } catch (InterruptedException e) { e.printStackTrace(); } } System.out.println("end_dire:" + dir.getPath()); }else{ System.out.println("file:" + dir.getPath()); if(visitor != null){ visitor.visitor(dir); } } this.isSuccess = true; } catch (Throwable e) { this.isSuccess = false; e.printStackTrace(); } this.endTime = new Date(); } //省略get set method }2、文件处理任务调度线程
package com.godway.filescanner; import java.io.File; import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; /** * 负责分发文件 * @author 高伟 * @date Jan 9, 2012 6:28:15 PM * @description: */ public class FileProcessor implements Runnable { private IScannerVisitor visitor; private ExecutorService pools; public FileProcessor(IScannerVisitor visitor) { this.visitor = visitor; /** * 使用无大小限制的连接池 * 通过下边任务控制,完成线程数量的控制 */ this.pools = Executors.newCachedThreadPool(); } @Override public void run() { while(true){ List<File> fileList = (List<File>) visitor.getScannerResult(); synchronized (fileList) { if (fileList.size() == 0) { try { /** * 处理完,没有待处理的,等待 */ fileList.wait(); } catch (InterruptedException e) { e.printStackTrace(); } } } /** * 根据要处理的任务数,控制线程数量,避免过多性能上的消耗 * 避免高峰请求等因素 */ if(fileList.size() <= 100){ scheduleTasks(20); }else if(fileList.size()>100){ scheduleTasks(80); }else if(fileList.size()>1000){ scheduleTasks(100); }else if(fileList.size()>2000){ scheduleTasks(500); } } } private void scheduleTasks(int count){ for (int j = 0; j < count; j++) { pools.submit(new FileProceTask((List<File>)visitor.getScannerResult())); } } }3、文件处理任务单元
package com.godway.filescanner; import java.io.File; import java.util.List; /** * 单个文件处理单元 * @author 高伟 * @date Jan 9, 2012 6:28:31 PM * @description: */ public class FileProceTask implements Runnable { private List<File> fileList; public FileProceTask(List<File> fileList) { this.fileList = fileList; } @Override public void run() { synchronized(fileList){ if(fileList.size() == 0){ try { fileList.wait(); } catch (InterruptedException e) { e.printStackTrace(); } } } File file = fileList.remove(fileList.size() - 1); System.out.println("process file : " + file.getPath()); /** * db operation etc. * ... */ } }
4、扫描器访问者接口
5、扫描器访问者实现之文件访问者
6、扫描任务测试类
/** * */ package com.godway.filescanner; import java.io.File; /** * @author 高伟 * @date Jan 9, 2012 5:22:32 PM * @description: */ public class DirectoryScannerTest { public static void main(String[] args) { File baseDir = new File("E:/work/"); FileScannerVisitor visitor = new FileScannerVisitor(); /** * 启动扫描线程 */ DirectoryScanner scanner = new DirectoryScanner(baseDir, visitor); Thread thread = new Thread(scanner); thread.start(); /** * 等待扫描完成 */ try { thread.join(); } catch (InterruptedException e) { e.printStackTrace(); } /** * 打印扫描结果 */ System.out.println(); System.out.println("=-=-=-=-=-=-=-=-=结果-=-=-=-=-=-=-=-=-="); System.out.println("Directory:" + scanner.getDir().getPath()); System.out.println("CreateTime:"+scanner.getCreateTime()); System.out.println("StartTime:"+scanner.getStartTime()); System.out.println("EndTime:"+scanner.getEndTime()); System.out.println("FileNum:"+visitor.getFileList().size()); System.out.println("=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-="); } }
7、扫描加处理综合测试类
8、源码打包下载地址: