nio按行读文件

最新推荐文章于 2023-08-07 17:37:00 发布

yunsyz

最新推荐文章于 2023-08-07 17:37:00 发布

阅读量1k

点赞数

分类专栏： java 文章标签： nio 按行读文件

本文链接：https://blog.csdn.net/yunsyz/article/details/72579546

版权

java 专栏收录该内容

14 篇文章 0 订阅

订阅专栏

不废话，直接上代码

package com.syz.test.nio;

import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;


public class DataReader {
    /**
     * 按行读，行的分隔符必须是单个字符"\n"
     */
    public static void readByLine(String filePath,String encoding,int bufferSize) {
        byte lf = 10;//"\n"
        RandomAccessFile raf = null;
        FileChannel fc = null;
        try {
            raf = new RandomAccessFile(filePath, "r");
            fc = raf.getChannel();
            ByteBuffer buffer = ByteBuffer.allocate(bufferSize);
            //保存换行后未处理的数据
            byte[] temp = new byte[0];
            while (fc.read(buffer)!= -1) {
                int position = buffer.position();// 读取结束后的位置，相当于读取的长度
                byte[] bs = new byte[position];// 用来存放读取的内容的数组
                buffer.rewind();// 将position设回0,所以你可以重读Buffer中的所有数据,此处如果不设置,无法使用下面的get方法
                buffer.get(bs);// 相当于buffer.get(bs,0,bs.length())：从position初始位置开始相对读,读bs.length个byte,并写入bs[0]到bs[bs.length-1]的区域
                buffer.clear();
                //要加上上次未处理的数据
                int readSize = position + temp.length;
                byte[] readByte = new byte[readSize];
                System.arraycopy(temp, 0, readByte, 0, temp.length);
                System.arraycopy(bs, 0, readByte, temp.length, position);
                int start = 0;
                int lineByteSize = 0;
                byte[] lineByte = new byte[0];
                for (int i = 0; i < readSize; i++) {
                    if (readByte[i] == lf) {
                        lineByte = new byte[lineByteSize];
                        System.arraycopy(readByte, start, lineByte, 0, lineByteSize);
                        String lineStr = new String(lineByte, encoding);
                        System.out.println(lineStr);
                        lineByteSize=0;
                        start = i+1;
                    } else {
                        lineByteSize++;
                    }

                }
                // 把剩余的放入临时数组中
                temp = new byte[lineByteSize];
                System.arraycopy(readByte, start, temp, 0, lineByteSize);
            }
            //处理临时数组中的数据
            if(temp.length>0){
                String lineStr = new String(temp, encoding);
                System.out.println(lineStr);
            }
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            if (fc != null) {
                try {
                    fc.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
            if (raf != null) {
                try {
                    raf.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
    }
    /**
     * 按条读
     * 举例：abc\ndef
     * 分隔符可以是多个字符，如：abc\n\rdedf
     * 如果设置成\n，就是按行读了
     * @param filePath 文件路径
     * @param encoding 字符集
     * @param bufferSize 每次读多少字节
     * @param itemSplit 第条的分隔符
     */
    public static void readByItem(String filePath,String encoding,int bufferSize,String itemSplit) {
        RandomAccessFile raf = null;
        FileChannel fc = null;
        try {
            raf = new RandomAccessFile(filePath, "r");
            fc = raf.getChannel();
            ByteBuffer buffer = ByteBuffer.allocate(bufferSize);
            byte[] itemSplitByte = itemSplit.getBytes(encoding);
            int itemSplitLen = itemSplitByte.length;
            //保存未处理的数据
            byte[] temp = new byte[0];
            int itemIndex = 0;
            while (fc.read(buffer)!= -1) {
                int position = buffer.position();// 读取结束后的位置，相当于读取的长度
                byte[] bs = new byte[position];// 用来存放读取的内容的数组
                buffer.rewind();// 将position设回0,所以你可以重读Buffer中的所有数据,此处如果不设置,无法使用下面的get方法
                buffer.get(bs);// 相当于buffer.get(bs,0,bs.length())：从position初始位置开始相对读,读bs.length个byte,并写入bs[0]到bs[bs.length-1]的区域
                buffer.clear();
                //要加上上次未处理的数据
                int readSize = position + temp.length;
                byte[] readByte = new byte[readSize];
                System.arraycopy(temp, 0, readByte, 0, temp.length);
                System.arraycopy(bs, 0, readByte, temp.length, position);
                int start = 0;
                int columnByteSize = 0;
                byte[] columnByte = new byte[0];
                int itemSplitIndex = 0;
                for (int i = 0; i < readSize; i++) {
                    columnByteSize++;
                    if (readByte[i] == itemSplitByte[itemSplitIndex]) {
                        itemSplitIndex++;
                        if(itemSplitIndex==itemSplitLen){
                            columnByteSize = columnByteSize-itemSplitLen;
                            columnByte = new byte[columnByteSize];
                            System.arraycopy(readByte, start, columnByte, 0, columnByteSize);
                            String column = new String(columnByte, encoding);
                            System.out.println(itemIndex+"\t"+column);
                            itemIndex++;
                            start = i + 1;
                            columnByteSize = 0;
                            itemSplitIndex = 0;
                        }
                    } else {
                        itemSplitIndex=0;
                    }

                }
                // 把剩余的放入临时数组中
                temp = new byte[columnByteSize];
                System.arraycopy(readByte, start, temp, 0, columnByteSize);
            }
            //处理临时数组中的数据
            System.out.println("temp.length="+temp.length);
            if(temp.length>0){
                String column = new String(temp, encoding);
                System.out.println(itemIndex+"\t"+column);
            }
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            if (fc != null) {
                try {
                    fc.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
            if (raf != null) {
                try {
                    raf.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
    }
    /**
     * 按条读，每一条又有被分成若干个列
     * 举例：a|b|\nc|d\n
     * 分隔符可以是多个字符，如：a::b::c\n\rd::e::f\n\r
     * 严重声明：条分隔符与列分隔符不能有一样的符号，如columnSplit=|,itemSplit=||，它们都有|符号。
     * 因为这个是逐个读的，不太好区分读到||的第一个字符时要不要切分列
     * @param filePath 文件路径
     * @param encoding 字符集
     * @param bufferSize 每次读多少字节
     * @param itemSplit 第条的分隔符
     * @param columnSplit 每条中每列的分隔符
     */
    public static void readByItem(String filePath,String encoding,int bufferSize,String itemSplit,String columnSplit) {
        RandomAccessFile raf = null;
        FileChannel fc = null;
        try {
            raf = new RandomAccessFile(filePath, "r");
            fc = raf.getChannel();
            ByteBuffer buffer = ByteBuffer.allocate(bufferSize);
            byte[] columnSplitByte = columnSplit.getBytes(encoding);
            int columnSplitLen = columnSplitByte.length;
            byte[] itemSplitByte = itemSplit.getBytes(encoding);
            int itemSplitLen = itemSplitByte.length;
            //保存未处理的数据
            byte[] temp = new byte[0];
            int columnIndex = 0;
            int itemIndex = 0;
            while (fc.read(buffer)!= -1) {
                int position = buffer.position();// 读取结束后的位置，相当于读取的长度
                byte[] bs = new byte[position];// 用来存放读取的内容的数组
                buffer.rewind();// 将position设回0,所以你可以重读Buffer中的所有数据,此处如果不设置,无法使用下面的get方法
                buffer.get(bs);// 相当于buffer.get(bs,0,bs.length())：从position初始位置开始相对读,读bs.length个byte,并写入bs[0]到bs[bs.length-1]的区域
                buffer.clear();
                //要加上上次未处理的数据
                int readSize = position + temp.length;
                byte[] readByte = new byte[readSize];
                System.arraycopy(temp, 0, readByte, 0, temp.length);
                System.arraycopy(bs, 0, readByte, temp.length, position);
                int start = 0;
                int columnByteSize = 0;
                byte[] columnByte = new byte[0];
                int columnSplitIndex = 0;
                int itemSplitIndex = 0;
                for (int i = 0; i < readSize; i++) {
                    columnByteSize++;
                    if (readByte[i] == columnSplitByte[columnSplitIndex]) {
                        columnSplitIndex++;
                        if(columnSplitIndex==columnSplitLen){
                            columnByteSize = columnByteSize-columnSplitLen;
                            columnByte = new byte[columnByteSize];
                            System.arraycopy(readByte, start, columnByte, 0, columnByteSize);
                            String column = new String(columnByte, encoding);
                            System.out.println(columnIndex+"\t"+column);
                            columnIndex++;
                            start = i + 1;
                            columnByteSize = 0;
                            columnSplitIndex = 0;
                        }
                    } else {
                        columnSplitIndex = 0;
                    }
                    if (readByte[i] == itemSplitByte[itemSplitIndex]) {
                        itemSplitIndex++;
                        if(itemSplitIndex==itemSplitLen){
                            columnByteSize = columnByteSize-itemSplitLen;
                            columnByte = new byte[columnByteSize];
                            System.arraycopy(readByte, start, columnByte, 0, columnByteSize);
                            String column = new String(columnByte, encoding);
                            System.out.println(columnIndex+"\t"+column);
                            System.out.println(itemIndex+"==========");
                            columnIndex=0;
                            itemIndex++;
                            start = i + 1;
                            columnByteSize = 0;
                            itemSplitIndex = 0;
                        }
                    } else {
                        itemSplitIndex=0;
                    }

                }
                // 把剩余的放入临时数组中
                temp = new byte[columnByteSize];
                System.arraycopy(readByte, start, temp, 0, columnByteSize);
            }
            //处理临时数组中的数据
            System.out.println("temp.length="+temp.length);
            if(temp.length>0){
                String column = new String(temp, encoding);
                System.out.println(columnIndex+"\t"+column);
            }
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            if (fc != null) {
                try {
                    fc.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
            if (raf != null) {
                try {
                    raf.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
    }

    public static void main(String[] args) {
        long t1 = System.currentTimeMillis();
        String columnSplit = "|";
        String itemSplit = "\n";
        int bufferSize = 48;
        String encoding = "GBK";
        String filePath = "d:/data/a1.txt";
        //readByLine(filePath,encoding,bufferSize);
        //readByItem(filePath,encoding,bufferSize,itemSplit);
        readByItem(filePath,encoding,bufferSize,itemSplit,columnSplit);
        long t2 = System.currentTimeMillis();
        System.out.println("time==="+(t2-t1));
    }
}

写文件

package com.syz.test.nio;

import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.text.SimpleDateFormat;
import java.util.Date;

public class DataWriter {
    public static void main(String[] args) {
        long t1 = System.currentTimeMillis();
        write();
        long t2 = System.currentTimeMillis();
        System.out.println("time==="+(t2-t1));
    }

    public static void write() {
        SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmmss");
        String date = sdf.format(new Date());
        StringBuffer sb = new StringBuffer();
        int bufferSize = 1024;
        String encoding = "UTF-8";
        String filePath = "d:/data/a_"+date+".txt";
        RandomAccessFile raf = null;
        FileChannel fc = null;
        try {
            raf = new RandomAccessFile(filePath, "rws");
            fc = raf.getChannel();
            ByteBuffer buffer = ByteBuffer.allocate(bufferSize);
            int count = 0;
            for(int i=0;i<10;i++){
                sb = new StringBuffer();
                sb.append("20170410_"+i);
                sb.append("|");
                sb.append("127.0.0.1");
                sb.append("|");
                sb.append("测试中文");
                sb.append("|");
                sb.append("www.xxx.com"+date+"_"+i);
                sb.append("|");
                sb.append("");
                sb.append("|");
                sb.append("");
                sb.append("|");
                sb.append("测试中文222abc");
                sb.append("|");
                sb.append("测ha");
                sb.append("|");
                sb.append("");
                sb.append("|");
                sb.append("2017-03-02");
                sb.append("|");
                sb.append("2017-03-02");
                sb.append("|");
                sb.append("2017-03-02");
                sb.append("|");
                sb.append("22haa测试");
                sb.append("|");
                sb.append("abc123");
                sb.append("\n");
                String data = sb.toString();
                byte[] dataByte = data.getBytes(encoding);
                int dataLen = dataByte.length;
                for(int j=0;j<dataLen;j++){
                    if(count==bufferSize){
                        buffer.flip();
                        int a  = fc.write(buffer);
                        System.out.println("a="+a);
                        buffer.clear();
                        count=0;
                    }
                    buffer.put(dataByte[j]);
                    count ++;
                }
            }
            buffer.flip();
            //写入最后一点儿
            int b = fc.write(buffer);
            System.out.println("b="+b);
        } catch (Exception e) {
            e.printStackTrace();
        }finally{
            if(fc!=null){
                try {
                    fc.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
            if(raf!=null){
                try {
                    raf.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
    }
}