文件扫描处理程序，用到线程、线程池、模板设计模式、访问者模式、类的递归等知识...

1、文件扫描任务线程

/** * */ package com.godway.filescanner; import java.io.File; import java.util.Date; /** * 递归调用 * 文件夹的文件扫描 * * @author 高伟 * @date Jan 9, 2012 11:16:39 AM * @description: */ public class DirectoryScanner implements Runnable { private File dir; private Date createTime;//创建时间 private Date startTime;//开始时间 private Date endTime;//结束时间 private boolean isSuccess;//是否成功 private IScannerVisitor visitor; public DirectoryScanner(File dir) { this.dir = dir; this.createTime = new Date(); this.startTime = new Date(); this.endTime = new Date(); this.isSuccess = false; } public DirectoryScanner(File dir, IScannerVisitor visitor) { this(dir); this.visitor = visitor; } @Override public void run() { this.startTime = new Date(); try { if(dir.isDirectory()){ System.out.println("start_dire:" + dir.getPath()); File[] fileList = dir.listFiles(); for (int i = 0; i < fileList.length; i++) { File file = fileList[i]; DirectoryScanner sub = new DirectoryScanner(file,visitor); Thread subThread = new Thread(sub); subThread.start(); try { /** * join关键字： * 子线程增加到主线程中， * 保证所有子线程结束后，才结束主线程 */ subThread.join(); } catch (InterruptedException e) { e.printStackTrace(); } } System.out.println("end_dire:" + dir.getPath()); }else{ System.out.println("file:" + dir.getPath()); if(visitor != null){ visitor.visitor(dir); } } this.isSuccess = true; } catch (Throwable e) { this.isSuccess = false; e.printStackTrace(); } this.endTime = new Date(); } //省略get set method }

2、文件处理任务调度线程

package com.godway.filescanner; import java.io.File; import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; /** * 负责分发文件 * @author 高伟 * @date Jan 9, 2012 6:28:15 PM * @description: */ public class FileProcessor implements Runnable { private IScannerVisitor visitor; private ExecutorService pools; public FileProcessor(IScannerVisitor visitor) { this.visitor = visitor; /** * 使用无大小限制的连接池 * 通过下边任务控制，完成线程数量的控制 */ this.pools = Executors.newCachedThreadPool(); } @Override public void run() { while(true){ List<File> fileList = (List<File>) visitor.getScannerResult(); synchronized (fileList) { if (fileList.size() == 0) { try { /** * 处理完，没有待处理的，等待 */ fileList.wait(); } catch (InterruptedException e) { e.printStackTrace(); } } } /** * 根据要处理的任务数，控制线程数量，避免过多性能上的消耗 * 避免高峰请求等因素 */ if(fileList.size() <= 100){ scheduleTasks(20); }else if(fileList.size()>100){ scheduleTasks(80); }else if(fileList.size()>1000){ scheduleTasks(100); }else if(fileList.size()>2000){ scheduleTasks(500); } } } private void scheduleTasks(int count){ for (int j = 0; j < count; j++) { pools.submit(new FileProceTask((List<File>)visitor.getScannerResult())); } } }

3、文件处理任务单元

package com.godway.filescanner; import java.io.File; import java.util.List; /** * 单个文件处理单元 * @author 高伟 * @date Jan 9, 2012 6:28:31 PM * @description: */ public class FileProceTask implements Runnable { private List<File> fileList; public FileProceTask(List<File> fileList) { this.fileList = fileList; } @Override public void run() { synchronized(fileList){ if(fileList.size() == 0){ try { fileList.wait(); } catch (InterruptedException e) { e.printStackTrace(); } } } File file = fileList.remove(fileList.size() - 1); System.out.println("process file : " + file.getPath()); /** * db operation etc. * ... */ } }

4、扫描器访问者接口

/** * */ package com.godway.filescanner; /** * 扫描器访问者接口 * @author 高伟 * @date Jan 9, 2012 5:43:39 PM * @description: */ public interface IScannerVisitor { /** * * visitor * @param obj * 高伟 * Jan 9, 2012 7:13:59 PM */ public void visitor(Object obj); /** * 返回扫描结果 * getScannerResult * @return * 高伟 * Jan 9, 2012 7:13:35 PM */ public Object getScannerResult(); }

5、扫描器访问者实现之文件访问者

/** * */ package com.godway.filescanner; import java.io.File; import java.util.LinkedList; import java.util.List; /** * 扫描器之访问者实现--文件访问者 * @author 高伟 * @date Jan 9, 2012 5:43:12 PM * @description: */ public class FileScannerVisitor implements IScannerVisitor { private List<File> fileList = new LinkedList<File>(); /* * (non-Javadoc) * @see com.godway.filescanner.IScannerVisitor#visitor(java.lang.Object) */ @Override public void visitor(Object obj) { if(obj instanceof File){ File file = (File)obj; synchronized(fileList){ if(!fileList.contains(file)){ fileList.add(file); } fileList.notify(); } } } /* * (non-Javadoc) * @see com.godway.filescanner.IScannerVisitor#getScannerResult() */ @Override public Object getScannerResult() { return fileList; } //省略 get set method }

6、扫描任务测试类

/** * */ package com.godway.filescanner; import java.io.File; /** * @author 高伟 * @date Jan 9, 2012 5:22:32 PM * @description: */ public class DirectoryScannerTest { public static void main(String[] args) { File baseDir = new File("E:/work/"); FileScannerVisitor visitor = new FileScannerVisitor(); /** * 启动扫描线程 */ DirectoryScanner scanner = new DirectoryScanner(baseDir, visitor); Thread thread = new Thread(scanner); thread.start(); /** * 等待扫描完成 */ try { thread.join(); } catch (InterruptedException e) { e.printStackTrace(); } /** * 打印扫描结果 */ System.out.println(); System.out.println("=-=-=-=-=-=-=-=-=结果-=-=-=-=-=-=-=-=-="); System.out.println("Directory:" + scanner.getDir().getPath()); System.out.println("CreateTime:"+scanner.getCreateTime()); System.out.println("StartTime:"+scanner.getStartTime()); System.out.println("EndTime:"+scanner.getEndTime()); System.out.println("FileNum:"+visitor.getFileList().size()); System.out.println("=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-="); } }

7、扫描加处理综合测试类

package com.godway.filescanner; import java.io.File; public class FileProcessorTest { public static void main(String[] args) { File baseDir = new File("E:/work"); FileScannerVisitor visitor = new FileScannerVisitor(); /** * 启动处理线程 */ FileProcessor processor = new FileProcessor(visitor); Thread procThread = new Thread(processor); procThread.start(); /** * 启动扫描线程 */ DirectoryScanner scanner = new DirectoryScanner(baseDir, visitor); Thread scanThread = new Thread(scanner); scanThread.start(); /** * 等待扫描完成 */ try { scanThread.join(); } catch (InterruptedException e) { e.printStackTrace(); } /** * 打印扫描结果 */ System.out.println(); System.out.println("=-=-=-=-=-=-=-=-=结果-=-=-=-=-=-=-=-=-="); System.out.println("Directory:" + scanner.getDir().getPath()); System.out.println("CreateTime:"+scanner.getCreateTime()); System.out.println("StartTime:"+scanner.getStartTime()); System.out.println("EndTime:"+scanner.getEndTime()); } }

8、源码打包下载地址：

文件扫描源程序打包下载