packagecom.chillax.imp;importjava.io.File;importjava.io.IOException;importjava.io.RandomAccessFile;importjava.nio.ByteBuffer;importjava.nio.channels.FileChannel;importjava.util.ArrayList;importjava.util.Date;importjava.util.List;/*** NIO读取百万级别文件
*@authorChillax
**/
public classNIO {public static void main(String args[]) throwsException {int bufSize = 1000000;//一次读取的字节长度
File fin = new File("D:\\test\\20160622_627975.txt");//读取的文件
File fout = new File("D:\\test\\20160622_627975_1.txt");//写出的文件
Date startDate = newDate();
FileChannel fcin= new RandomAccessFile(fin, "r").getChannel();
ByteBuffer rBuffer=ByteBuffer.allocate(bufSize);
FileChannel fcout= new RandomAccessFile(fout, "rws").getChannel();
ByteBuffer wBuffer=ByteBuffer.allocateDirect(bufSize);
readFileByLine(bufSize, fcin, rBuffer, fcout, wBuffer);
Date endDate= newDate();
System.out.print(startDate+"|"+endDate);//测试执行时间
if(fcin.isOpen()){
fcin.close();
}if(fcout.isOpen()){
fcout.close();
}
}public static void readFileByLine(intbufSize, FileChannel fcin,
ByteBuffer rBuffer, FileChannel fcout, ByteBuffer wBuffer) {
String enter= "\n";
List dataList = new ArrayList();//存储读取的每行数据
byte[] lineByte = new byte[0];
String encode= "GBK";//String encode = "UTF-8";
try{//temp:由于是按固定字节读取,在一次读取中,第一行和最后一行经常是不完整的行,因此定义此变量来存储上次的最后一行和这次的第一行的内容,//并将之连接成完成的一行,否则会出现汉字被拆分成2个字节,并被提前转换成字符串而乱码的问题
byte[] temp = new byte[0];while (fcin.read(rBuffer) != -1) {//fcin.read(rBuffer):从文件管道读取内容到缓冲区(rBuffer)
int rSize = rBuffer.position();//读取结束后的位置,相当于读取的长度
byte[] bs = new byte[rSize];//用来存放读取的内容的数组
rBuffer.rewind();//将position设回0,所以你可以重读Buffer中的所有数据,此处如果不设置,无法使用下面的get方法
rBuffer.get(bs);//相当于rBuffer.get(bs,0,bs.length()):从position初始位置开始相对读,读bs.length个byte,并写入bs[0]到bs[bs.length-1]的区域
rBuffer.clear();int startNum = 0;int LF = 10;//换行符
int CR = 13;//回车符
boolean hasLF = false;//是否有换行符
for(int i = 0; i < rSize; i++){if(bs[i] ==LF){
hasLF= true;int tempNum =temp.length;int lineNum = i -startNum;
lineByte= new byte[tempNum + lineNum];//数组大小已经去掉换行符
System.arraycopy(temp,0, lineByte, 0, tempNum);//填充了lineByte[0]~lineByte[tempNum-1]
temp = new byte[0];
System.arraycopy(bs, startNum, lineByte, tempNum, lineNum);//填充lineByte[tempNum]~lineByte[tempNum+lineNum-1]
String line= new String(lineByte, 0, lineByte.length, encode);//一行完整的字符串(过滤了换行和回车)
dataList.add(line);//System.out.println(line);
writeFileByLine(fcout, wBuffer, line +enter);//过滤回车符和换行符
if(i + 1 < rSize && bs[i + 1] ==CR){
startNum= i + 2;
}else{
startNum= i + 1;
}
}
}if(hasLF){
temp= new byte[bs.length -startNum];
System.arraycopy(bs, startNum, temp,0, temp.length);
}else{//兼容单次读取的内容不足一行的情况
byte[] toTemp = new byte[temp.length +bs.length];
System.arraycopy(temp,0, toTemp, 0, temp.length);
System.arraycopy(bs,0, toTemp, temp.length, bs.length);
temp=toTemp;
}
}if(temp != null && temp.length > 0){//兼容文件最后一行没有换行的情况
String line = new String(temp, 0, temp.length, encode);
dataList.add(line);//System.out.println(line);
writeFileByLine(fcout, wBuffer, line +enter);
}
}catch(IOException e) {
e.printStackTrace();
}
}/*** 写到文件上
*@paramfcout
*@paramwBuffer
*@paramline*/@SuppressWarnings("static-access")public static voidwriteFileByLine(FileChannel fcout, ByteBuffer wBuffer,
String line) {try{
fcout.write(wBuffer.wrap(line.getBytes("UTF-8")), fcout.size());
}catch(IOException e) {
e.printStackTrace();
}
}
}