文件扫描处理程序，用到线程、线程池、模板设计模式、访问者模式、类的递归等知识

1、文件扫描任务线程

/**
 * 
 */
package com.godway.filescanner;

import java.io.File;
import java.util.Date;

/**
 * 递归调用
 * 文件夹的文件扫描
 * 
 * @author 高伟
 * @date Jan 9, 2012 11:16:39 AM
 * @description:
 */
public class DirectoryScanner implements Runnable {
	private File dir;
	
	private Date createTime;//创建时间
	private Date startTime;//开始时间
	private Date endTime;//结束时间
	private boolean isSuccess;//是否成功
	
	private IScannerVisitor visitor;
	
	public DirectoryScanner(File dir) {
		this.dir = dir;
		
		this.createTime = new Date();
		this.startTime = new Date();
		this.endTime = new Date();
		this.isSuccess = false;
	}
	
	public DirectoryScanner(File dir, IScannerVisitor visitor) {
		this(dir);
		
		this.visitor = visitor;
	}
	
	@Override
	public void run() {
		this.startTime = new Date();
		try {
			if(dir.isDirectory()){
				System.out.println("start_dire:" + dir.getPath());
				File[] fileList = dir.listFiles();
				for (int i = 0; i < fileList.length; i++) {
					File file = fileList[i];
					DirectoryScanner sub = new DirectoryScanner(file,visitor);
					Thread subThread = new Thread(sub);
					subThread.start();
					try {
						/**
						 * join关键字：
						 * 子线程增加到主线程中，
						 * 保证所有子线程结束后，才结束主线程
						 */
						subThread.join();
					} catch (InterruptedException e) {
						e.printStackTrace();
					}
				}
				System.out.println("end_dire:" + dir.getPath());
			}else{
				System.out.println("file:" + dir.getPath());
				if(visitor != null){
					visitor.visitor(dir);
				}
			}
			this.isSuccess = true;
		} catch (Throwable e) {
			this.isSuccess = false;
			e.printStackTrace();
		}
		this.endTime = new Date();
	}

	//省略get set method	
}

2、文件处理任务调度线程

package com.godway.filescanner;

import java.io.File;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

/**
 * 负责分发文件
 * @author 高伟
 * @date Jan 9, 2012 6:28:15 PM
 * @description:
 */
public class FileProcessor implements Runnable {
	private IScannerVisitor visitor;
	private ExecutorService pools;
	
	public FileProcessor(IScannerVisitor visitor) {
		this.visitor = visitor;
		/**
		 * 使用无大小限制的连接池
		 * 通过下边任务控制，完成线程数量的控制
		 */
		this.pools = Executors.newCachedThreadPool();
	}

	@Override
	public void run() {
		while(true){
			List<File> fileList = (List<File>) visitor.getScannerResult();
			synchronized (fileList) {
				if (fileList.size() == 0) {
					try {
						/**
						 * 处理完，没有待处理的，等待
						 */
						fileList.wait();
					} catch (InterruptedException e) {
						e.printStackTrace();
					}
				}
			}
			/**
			 * 根据要处理的任务数，控制线程数量，避免过多性能上的消耗
			 * 避免高峰请求等因素
			 */
			if(fileList.size() <= 100){
				scheduleTasks(20);
			}else if(fileList.size()>100){
				scheduleTasks(80);
			}else if(fileList.size()>1000){
				scheduleTasks(100);
			}else if(fileList.size()>2000){
				scheduleTasks(500);
			}
		}
		
	}
	
	private void scheduleTasks(int count){
		for (int j = 0; j < count; j++) {
			pools.submit(new FileProceTask((List<File>)visitor.getScannerResult()));
		}
	}
	
}

3、文件处理任务单元

package com.godway.filescanner;

import java.io.File;
import java.util.List;

/**
 * 单个文件处理单元
 * @author 高伟
 * @date Jan 9, 2012 6:28:31 PM
 * @description:
 */
public class FileProceTask implements Runnable {
	private List<File> fileList;
	
	public FileProceTask(List<File> fileList) {
		this.fileList = fileList;
	}

	@Override
	public void run() {
		synchronized(fileList){
			if(fileList.size() == 0){
				try {
					fileList.wait();
				} catch (InterruptedException e) {
					e.printStackTrace();
				}
			}
		}
		File file = fileList.remove(fileList.size() - 1);
		System.out.println("process file : " + file.getPath());
		/**
		 * db operation etc.
		 * ...
		 */
	}
	
}

4、扫描器访问者接口

/**
 * 
 */
package com.godway.filescanner;

/**
 * 扫描器访问者接口
 * @author 高伟
 * @date Jan 9, 2012 5:43:39 PM
 * @description:
 */
public interface IScannerVisitor {
	
	/**
	 * 
	 * visitor
	 * @param obj
	 * 高伟
	 * Jan 9, 2012 7:13:59 PM
	 */
	public void visitor(Object obj);
	
	/**
	 * 返回扫描结果
	 * getScannerResult
	 * @return
	 * 高伟
	 * Jan 9, 2012 7:13:35 PM
	 */
	public Object getScannerResult();
}

5、扫描器访问者实现之文件访问者

/**
 * 
 */
package com.godway.filescanner;

import java.io.File;
import java.util.LinkedList;
import java.util.List;

/**
 * 扫描器之访问者实现--文件访问者
 * @author 高伟
 * @date Jan 9, 2012 5:43:12 PM
 * @description:
 */
public class FileScannerVisitor implements IScannerVisitor {

	private List<File> fileList = new LinkedList<File>();
	
	/*
	 * (non-Javadoc)
	 * @see com.godway.filescanner.IScannerVisitor#visitor(java.lang.Object)
	 */
	@Override
	public void visitor(Object obj) {
		if(obj instanceof File){
			File file = (File)obj;
			synchronized(fileList){
				if(!fileList.contains(file)){
					fileList.add(file);
				}
				fileList.notify();
			}
		}
	}

	/*
	 * (non-Javadoc)
	 * @see com.godway.filescanner.IScannerVisitor#getScannerResult()
	 */
	@Override
	public Object getScannerResult() {
		return fileList;
	}

	//省略 get  set method	
}

6、扫描任务测试类

/**
 * 
 */
package com.godway.filescanner;

import java.io.File;


/**
 * @author 高伟
 * @date Jan 9, 2012 5:22:32 PM
 * @description:
 */
public class DirectoryScannerTest {

	public static void main(String[] args) {
		File baseDir = new File("E:/work/");
		FileScannerVisitor visitor = new FileScannerVisitor();
		
		/**
		 * 启动扫描线程
		 */
		DirectoryScanner scanner = new DirectoryScanner(baseDir, visitor);
		Thread thread = new Thread(scanner);
		thread.start();
		
		/**
		 * 等待扫描完成
		 */
		try {
			thread.join();
		} catch (InterruptedException e) {
			e.printStackTrace();
		}
		
		/**
		 * 打印扫描结果
		 */
		System.out.println();
		System.out.println("=-=-=-=-=-=-=-=-=结果-=-=-=-=-=-=-=-=-=");
		System.out.println("Directory:" + scanner.getDir().getPath());
		System.out.println("CreateTime:"+scanner.getCreateTime());
		System.out.println("StartTime:"+scanner.getStartTime());
		System.out.println("EndTime:"+scanner.getEndTime());
		System.out.println("FileNum:"+visitor.getFileList().size());
		System.out.println("=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=");
	}
}

7、扫描加处理综合测试类

package com.godway.filescanner;

import java.io.File;

public class FileProcessorTest {
	public static void main(String[] args) {
		File baseDir = new File("E:/work");
		FileScannerVisitor visitor = new FileScannerVisitor();
		
		/**
		 * 启动处理线程
		 */
		FileProcessor processor = new FileProcessor(visitor);
		Thread procThread = new Thread(processor); 
		procThread.start();
		
		/**
		 * 启动扫描线程
		 */
		DirectoryScanner scanner = new DirectoryScanner(baseDir, visitor);
		Thread scanThread = new Thread(scanner);
		scanThread.start();
		
		/**
		 * 等待扫描完成
		 */
		try {
			scanThread.join();
		} catch (InterruptedException e) {
			e.printStackTrace();
		}
		
		/**
		 * 打印扫描结果
		 */
		System.out.println();
		System.out.println("=-=-=-=-=-=-=-=-=结果-=-=-=-=-=-=-=-=-=");
		System.out.println("Directory:" + scanner.getDir().getPath());
		System.out.println("CreateTime:"+scanner.getCreateTime());
		System.out.println("StartTime:"+scanner.getStartTime());
		System.out.println("EndTime:"+scanner.getEndTime());
		
	}
}

8、源码打包下载地址：

文件扫描源程序打包下载