Copyright ©2018
hbgengfei11
一、引言
使用java读写文件是日常工作中需要经常使用的技术。为了提高文件写出分析和写出效率,本文采用多线程实现多文件解析和写出。具体实现如下:
二、文件读写工具类
package com.test.multiThreadsReadFile.utils;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.Reader;
import java.io.UnsupportedEncodingException;
import java.io.Writer;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import com.univocity.parsers.csv.CsvWriter;
import com.univocity.parsers.csv.CsvWriterSettings;
/**
* @author admin 文件读写工具类
*/
public class FileUtil
{
/**
* 读取txt文件工具类
* @param inputPath
* @return
*/
public static Map<String,List<String>> readTxtFile(String inputPath)
{
Map<String,List<String>> map = new HashMap<String,List<String>>();
List<String> aList = new ArrayList<String>();
List<String> bList = new ArrayList<String>();
List<String> cList = new ArrayList<String>();
BufferedReader reader = null;
try
{
reader = new BufferedReader(new InputStreamReader(
new FileInputStream(inputPath), "UTF-8"),10*1024*1024);
String line = "";
while((line = reader.readLine()) != null)
{
if(null == line || line.isEmpty())
{
continue;
}
String[] split = line.split("=");
if(split.length < 2)
{
continue;
}
if(split[0].equals("a"))
{
aList.add(split[1]);
}
if(split[0].equals("b"))
{
bList.add(split[1]);
}
if(split[0].equals("c"))
{
cList.add(split[1]);
}
}
map.put("a", aList);
map.put("b", bList);
map.put("c", cList);
} catch (FileNotFoundException e)
{
e.printStackTrace();
} catch (UnsupportedEncodingException e)
{
e.printStackTrace();
} catch (IOException e)
{
e.printStackTrace();
}
return map;
}
/**
* 写出CSV文件工具类
*
* @param os
* @param headers
* @param rows
* @throws IOException
*/
public static void writerCSVFile(String outputPath, List<String> headers, List<Object[]> rows,String prefixName)
{
String fileName = prefixName + ".csv";
// 新建需要写出的文件
File file = new File(outputPath,fileName);
Writer writer = null;
try
{
// 如果写出地址是个文件夹,创建新文件
if (file.isDirectory())
{
file.createNewFile();
}
writer = new FileWriter(file, true);
// 创建文件写出流
// 创建CSV文件写出设置
CsvWriterSettings setting = new CsvWriterSettings();
// 写出文件,并可以追加
CsvWriter csvwriter = new CsvWriter(writer, setting);
// 写出文件标题
csvwriter.writeHeaders(headers);
// 写出文件内容
csvwriter.writeRowsAndClose(rows);
} catch (IOException e)
{
e.printStackTrace();
} finally
{
try
{
if (null != writer)
{
// 关闭文件写出流
writer.close();
}
} catch (IOException e)
{
e.printStackTrace();
}
}
}
}
三、业务接口类
package com.test.multiThreadsReadFile.service;
/**
* @author admin
* 文件读写接口
*/
public interface IService
{
/**
* 文件读取主方法
* @param inputPath
* @param outputPath
* @return
*/
long readFile(String inputPath,String outputPath);
}
四、业务实现类
package com.test.multiThreadsReadFile.service.impl;
import java.util.ArrayList;
import java.util.Hashtable;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import com.test.multiThreadsReadFile.service.IService;
import com.test.multiThreadsReadFile.utils.FileUtil;
public class TxtFileService implements IService
{
/*
* 读取文件,开启线程主方法
*/
@Override
public long readFile(String inputPath,String outputPath)
{
//统计解析成功条数
long successNum = 0;
//读取全部文件
Map<String, List<String>> readTxtFile = FileUtil.readTxtFile(inputPath);
//解析成功条数统计
Hashtable<String, Long> successCount = new Hashtable<String, Long>();
//获取标题
List<String> head = getHead();
//组装线程名字,既文件前缀
String[] fileName =
{ "a", "b", "c" };
// 创建计数器
// 构造参数传入的数量值代表的是latch.countDown()调用的次数
CountDownLatch latch = new CountDownLatch(3);
// 创建线程池,可以通过以下方式创建
ExecutorService threadPool = Executors.newFixedThreadPool(3);
for (int i = 0; i < fileName.length; i++)
{
threadPool.execute(new ReadTxtFileThread(readTxtFile, head, latch, fileName[i], successCount,outputPath));
}
try
{
// 阻塞当前线程,直到所有线程都执行完毕
latch.await();
} catch (InterruptedException e)
{
e.printStackTrace();
} finally
{
//所有业务执行完毕,关闭线程池
threadPool.shutdown();
}
//解析成功条数统计
for (Long num : successCount.values())
{
successNum += num;
}
return successNum;
}
/**
* 组装文件表头
* @return
*/
private List<String> getHead()
{
List<String> head = new ArrayList<String>();
head.add("A1");
head.add("A2");
head.add("A3");
return head;
}
}
五、解析与写出文件线程
package com.test.multiThreadsReadFile.service.impl;
import java.util.ArrayList;
import java.util.Hashtable;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
import com.test.multiThreadsReadFile.utils.FileUtil;
/**
* @author admin
* 文件写出线程
*/
public class ReadTxtFileThread implements Runnable
{
/**
* 读取来的数据
*/
private Map<String, List<String>> allLines;
/**
* 表头
*/
private List<String> head;
/**
* 线程计数
*/
private CountDownLatch latch;
/**
* 线程name
*/
private String ThreadName;
/**
* 文件写出地址
*/
private String outputPath;
/**
* 统计
*/
private Hashtable<String, Long> successCount;
public ReadTxtFileThread(Map<String, List<String>> allLines, List<String> head, CountDownLatch latch,
String threadName, Hashtable<String, Long> successCount,String outputPath)
{
super();
this.allLines = allLines;
this.head = head;
this.latch = latch;
this.ThreadName = threadName;
this.successCount = successCount;
this.outputPath = outputPath;
}
@Override
public void run()
{
//通过判断线程名字来决定执行那个线程
String[] str = {"a","b","c"};
for (String name : str)
{
if(name.equals(ThreadName))
{
writeTxtFile(name);
}
}
//线程计数
latch.countDown();
}
/**
* 解析并写出文件
* @param type
*/
private synchronized void writeTxtFile(String type)
{
//每个文件写出条数统计
long sucNum = 0;
//需要写出的行
List<Object[]> rows = new ArrayList<Object[]>();
//生成文件标题
String prefixName = type + System.currentTimeMillis();
//需要写出的文件的原始数据
List<String> list = allLines.get(type);
//根据需求解析文件
for (String line : list)
{
String[] split = line.split(",");
rows.add(split);
}
//统计单个文件成功条数
sucNum = rows.size();
successCount.put(type, sucNum);
//调用工具类,写出CSV文件
FileUtil.writerCSVFile(outputPath, head, rows,prefixName);
}
}
六、测试
package com.test.multiThreadsReadFile.test;
import com.test.multiThreadsReadFile.service.IService;
import com.test.multiThreadsReadFile.service.impl.TxtFileService;
/**
* @author admin
* 测试类
*/
public class MyTest
{
public static void main(String[] args)
{
//读取文件地址
String inputPath = "C:\\Users\\test\\Desktop\\demo.txt";
//写出文件地址
String outputPath = "C:\\Users\\test\\Desktop\\1\\";
IService readTxt = new TxtFileService();
long readFile = readTxt.readFile(inputPath, outputPath);
System.out.println("成功解析" + readFile + "条数据");
}
}
七、总结
本文中文件的解析和写出采用了多线,如果可以起三个线程:一个线程读、一个线程解析、一个线程写,三个线程同时工作可以大大提高读写效率。欢迎评论探讨!