在读取大文件的时候,采用管道方式进行分流,使用byteBuffer把文件分成一段段的进行读写。
生成大文件 :
读文件:
采用NI0可以加快文件的读取速度!
生成大文件 :
public class ProductionFile {
private static void productFile() throws FileNotFoundException {
File file = new File("D://larger.txt");
PrintWriter pw = new PrintWriter(file);
try{
for (int i = 0;i<1024;i++) {
for (int j = 0;j <1024;j++){
for(int k = 0;k<1024;i++){
pw.write(i+":" +j);
}
pw.flush();
}
pw.flush();
}
}finally{
pw.close();
}
}
public static void main(String[] args) {
try {
productFile();
} catch (FileNotFoundException e) {
e.printStackTrace();
}
}
}
读文件:
public class LargeMappedFiles {
static int length = 0x300000;
/**
* @param args
* @throws IOException
* @throws FileNotFoundException
*/
public static void main(String[] args) throws FileNotFoundException, IOException {
File file = new File("D://larger.txt");
FileChannel fileChannel = new RandomAccessFile(file, "rw")
.getChannel();
/**
* map(FileChannel.MapMode mode,long position, long size) mode -
* 根据是按只读、读取/写入或专用(写入时拷贝)来映射文件,分别为 FileChannel.MapMode 类中所定义的
* READ_ONLY、READ_WRITE 或 PRIVATE 之一 position - 文件中的位置,映射区域从此位置开始;必须为非负数
* size - 要映射的区域大小;必须为非负数且不大于 Integer.MAX_VALUE
* 所以若想读取文件后半部分内容,如例子所写;若想读取文本后1
* /8内容,需要这样写map(FileChannel.MapMode.READ_ONLY,
* f.length()*7/8,f.length()/8)
* 想读取文件所有内容,需要这样写map(FileChannel.MapMode.READ_ONLY, 0,f.length())
*/
MappedByteBuffer inputBuffer = fileChannel.map(FileChannel.MapMode.READ_ONLY, 0, file.length()/40);
long start = System.currentTimeMillis();
byte[] dst = new byte[length];
System.out.println("File size:" + inputBuffer.capacity());
for( int offSet = 0; offSet<inputBuffer.capacity(); offSet += length) {
if(inputBuffer.capacity() - offSet > length) {
for (int i = 0 ; i < length ; i++) {
dst[i] = inputBuffer.get(offSet + i );
if (i == 10)
System.out.println("i:" + i+ " Value-->"+new String(dst,0, 40)+ " " );
}
}else {
for (int i = 0 ; i< inputBuffer.capacity() - offSet; i++) {
byte b = inputBuffer.get(offSet + i );
}
}
}
long end = System.currentTimeMillis();
System.out.println("Value-->"+new String(dst, 9990, 10000)+ " " );
System.out.println(end -start );
// System.out.println(bb.get(count-10));
}
}
采用NI0可以加快文件的读取速度!
public class MappedIO {
private static int numOfInts = 4000000;
private static int numberOfUbuffInts = 200000;
private abstract static class Tester{
private String name;
public Tester(String name) {
this.name = name;
}
public void runTest() {
System.out.print(name + ":");
try{
long start = System.nanoTime();
test();
double duration = System.nanoTime() - start;
System.out.format("%.2f\n",duration/1.0e9);
}catch(IOException e){
throw new RuntimeException(e);
}
}
public abstract void test() throws IOException;
}
private static Tester[] tests = {
new Tester("Stream Write") {
public void test() throws IOException {
DataOutputStream dos = new DataOutputStream(
new BufferedOutputStream(new FileOutputStream(new File(
"temp.temp"))));
for(int i = 0 ;i<numOfInts ;i++){
dos.writeInt(i);
}
dos.close();
}
},
new Tester("Mapped Write") {
public void test() throws IOException {
FileChannel fc = new RandomAccessFile("temp.temp", "rw").getChannel();
IntBuffer ib = fc.map(FileChannel.MapMode.READ_WRITE, 0, fc.size()).asIntBuffer();
for(int i = 0; i<numOfInts; i++)
ib.put(i);
fc.close();
}
},
new Tester("Stream Reader") {
public void test() throws IOException {
DataInputStream dis = new DataInputStream(new BufferedInputStream(
new FileInputStream(new File("temp.temp"))));
for (int i = 0; i<numOfInts; i++) {
dis.readInt();
}
dis.close();
}
} ,
new Tester("Mapped Reader") {
public void test() throws IOException {
FileChannel fc = new RandomAccessFile("temp.temp","rw").getChannel();
IntBuffer ib = fc.map(FileChannel.MapMode.READ_WRITE, 0, fc.size()).asIntBuffer();
while(ib.hasRemaining()) {
ib.get();
}
fc.close();
}
},
new Tester("Stream Writer Reader") {
public void test() throws IOException {
RandomAccessFile raf = new RandomAccessFile(new File("temp.temp"),"rw");
raf.writeInt(1);
for(int i = 0; i<numberOfUbuffInts; i++){
raf.seek(raf.length()-4);
raf.writeInt(raf.readInt());
}
raf.close();
}
},
new Tester("Mapped Writer/Reader") {
public void test() throws IOException {
FileChannel fc = new RandomAccessFile(new File("temp.temp"), "rw").getChannel();
IntBuffer ib = fc.map(FileChannel.MapMode.READ_WRITE, 0, fc.size()).asIntBuffer();
ib.put(0);
for(int i=1;i<numberOfUbuffInts;i++) {
ib.put(ib.get(i-1));
}
fc.close();
}
}
};
public static void main(String args[]) throws IOException {
for (Tester test:tests) {
test.runTest();
}
}
}