java 分批读取文件_java多线程批量读取文件( 八)--读写分离

packagecom.net.thread.future;importjava.io.BufferedReader;importjava.io.BufferedWriter;importjava.io.File;importjava.io.FileInputStream;importjava.io.FileNotFoundException;importjava.io.FileWriter;importjava.io.IOException;importjava.io.InputStreamReader;importjava.text.SimpleDateFormat;importjava.util.ArrayList;importjava.util.Date;importjava.util.HashMap;importjava.util.Iterator;importjava.util.List;importjava.util.Map;importjava.util.Random;importjava.util.Set;importjava.util.concurrent.ArrayBlockingQueue;importjava.util.concurrent.BlockingQueue;importjava.util.concurrent.Callable;importjava.util.concurrent.CountDownLatch;importjava.util.concurrent.ExecutionException;importjava.util.concurrent.ExecutorService;importjava.util.concurrent.Executors;importjava.util.concurrent.Future;/***@author* @Time：2017年8月16日 下午5:26:37

*@version1.0

* @description*/

public classCallableDemo3 {final static SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");public static voidmain(String[] args)

{

File f= new File("C://Users//LENOVO//Desktop//file");//文件总数

final List filePathsList = new ArrayList();

File[] filePaths=f.listFiles();for(File s : filePaths) {

filePathsList.add(s);

}

CountDownLatch latch= newCountDownLatch(filePathsList.size());

ExecutorService pool= Executors.newFixedThreadPool(10);

BlockingQueue>> queue =

new ArrayBlockingQueue>>(100);

System.out.println("-------------文件读、写任务开始时间：" + sdf.format(newDate()));for (int i = 0; i < filePathsList.size(); i++) {

File temp=filePathsList.get(i);

Future> future = pool.submit(newMyCallableProducer(latch, temp));

queue.add(future);

pool.execute(newMyCallableConsumer(queue));

}try{

latch.await();

}catch(InterruptedException e) {

e.printStackTrace();

}

System.out.println("-------------文件读、写任务结束时间：" + sdf.format(newDate()));

pool.shutdownNow();

}//文件读线程

static class MyCallableProducer implements Callable>{privateCountDownLatch latch;privateFile file;private FileInputStream fis = null;private Map fileMap = new HashMap();publicMyCallableProducer(CountDownLatch latch, File file)

{this.latch =latch;this.file =file;

}

@Overridepublic Map call() throwsException

{

System.out.println(Thread.currentThread().getName()+ " 线程开始读取文件 ：" + file.getName() + " ,时间为 "+ sdf.format(newDate()));

fis= newFileInputStream(file);

fileMap.put(file.getName(), fis);

doWork();

System.out.println(Thread.currentThread().getName()+ " 线程读取文件 ：" + file.getName() + " 完毕" + " ,时间为 "+ sdf.format(newDate()));

latch.countDown();returnfileMap;

}private voiddoWork()

{

//此方法可以添加一些业务逻辑，比如包装pojo等等操作，返回的值可以是任何类型

Random rand= newRandom();int time = rand.nextInt(10) * 1000;try{

Thread.sleep(time);

}catch(InterruptedException e) {

e.printStackTrace();

}

}

}//文件写线程

static class MyCallableConsumer implementsRunnable

{private String fileName = "";private BlockingQueue>>queue;private FileInputStream fis = null;private File dirFile = null;private BufferedReader br = null;private InputStreamReader isr = null;private FileWriter fw = null;private BufferedWriter bw = null;public MyCallableConsumer(BlockingQueue>>queue2)

{this.queue =queue2;

}

@Overridepublic voidrun()

{try{

Future> future =queue.take();

Map map =future.get();

Set set =map.keySet();for (Iterator iter =set.iterator(); iter.hasNext();) {

fileName=iter.next().toString();

fis=map.get(fileName);

System.out.println(Thread.currentThread().getName()+ " 线程开始写文件 ：" + fileName + " ,时间为 "+ sdf.format(newDate()));try{

isr= new InputStreamReader(fis, "utf-8");

br= newBufferedReader(isr);

dirFile= new File("d:" + File.separator + "gc3" + File.separator +fileName);

fw= newFileWriter(dirFile);

bw= newBufferedWriter(fw);

String data= "";

bw.write("+++++++++++++" + Thread.currentThread().getName() + " 线程开始写文件++++++++++++");while ((data = br.readLine()) != null) {

bw.write(data+ "\r");

}

}catch(FileNotFoundException e) {

e.printStackTrace();

}catch(IOException e) {

e.printStackTrace();

}finally{try{

bw.close();

br.close();

}catch(IOException e) {

e.printStackTrace();

}

}

}

}catch(InterruptedException e) {

e.printStackTrace();

}catch(ExecutionException e) {

e.printStackTrace();

}

}

}

}