一、优化过程:
问题:所有文件读取慢。
优化1、之前:一个字节一个字节的读取。之后每次读取8*1024个字节。效率提高100倍。
问题:大文件读取慢。
优化2:之前内容用LinkedList,之后用ArrayList。LinkedList差寻慢,不是从头查就是从尾查
问题:读取大文件oom
原因:之前先一次性读取到StringBuilder(StringBuilder一开始被分配在young区,15次ygc后晋升到old区,晋升到old区的StringBuilder依然不断增大,直至oom)。已后期验证同样的数据StringBuilder将近比ArrayList大一倍。
优化3:之前先一次性读取到StringBuilder。之后每次读取8*1024个字节到StringBuilder(利用了young区的oom),同时加到ArrayList。
二、demo
package d;
import lombok.extern.slf4j.Slf4j;
import java.io.*;
import java.nio.charset.Charset;
import java.util.*;
import java.util.concurrent.CountDownLatch;
/**
* @author sunxuejian
* @date 2020/6/23
*/
@Slf4j
public class Test_IO {
public final static String PARAMETERIZED_FILE = "parameterized.properties";
protected static Charset charset = Charset.forName("utf-8");
static {
InputStream is = AbstractAsciiFileAccess.class.getClassLoader().getResourceAsStream(PARAMETERIZED_FILE);
Properties properties = new Properties();
try {
properties.load(is);
String encoding = properties.getProperty("file_encoding", "utf-8");
System.out.println("===========>encoding" + encoding);
if (!"".equals(encoding = encoding.trim())) {
charset = Charset.forName(encoding);
}
} catch (IOException e) {
e.printStackTrace();
}
}
protected RandomAccessFile randomAccessFile;
protected String separator;
protected String readMode = "r";
protected boolean enCycle;//是否回去读
protected long lineCount;
protected void setLineCount(File file) {
LineNumberReader rf = null;
try {
rf = new LineNumberReader(new FileReader(file));
if (rf != null) {
rf.skip(file.length());
this.lineCount = rf.getLineNumber() + 1;
}
} catch (IOException e) {
log.error(e.getMessage(), e);
} finally {
if (rf != null) {
try {
rf.close();
} catch (IOException e) {
log.error(e.getMessage(), e);
}
}
}
}
public String readLine() {
try {
return this.nextLine();
} catch (IOException e) {
e.printStackTrace();
}
return null;
}
private volatile Boolean isReaded = false;
private int readLineCount = 0;
List<String> lines = new ArrayList();
private synchronized String nextLine() throws IOException {
if(!isReaded) {
StringBuilder builder = new StringBuilder();
byte[] b = new byte[80 * 1024];
int i;
while(true) {
i=randomAccessFile.read(b);
if (i == -1) {
lines.add(builder.toString());
break;
}
builder.append(new String(b, 0, i, charset));
List<String> lineList = Arrays.asList(builder.toString().split(System.getProperty("line.separator")));
int length = lineList.size();
if (length == 1){
continue;
}
for (int j=0; j<length-1; j++){
lines.add(lineList.get(j));
}
builder = new StringBuilder(lineList.get(length-1));
}
isReaded = true;
}
if(readLineCount >= lineCount) {
if(enCycle){
readLineCount=0;
}else {
return null;
}
}
String s = lines.get(readLineCount++);
// System.out.println("====>" + s);
return s;
}
public Test_IO(String filepath) {
this(filepath, null, false);
}
public Test_IO(String filepath, String separator, boolean enCycle) {
try {
File file = new File(filepath);
if (file.exists() && file.isFile()) {
randomAccessFile = new RandomAccessFile(file, readMode);
} else {
file = new File(AsciiFileAccessCircularly_type2.class.getClassLoader().getResource(filepath).getFile());
randomAccessFile = new RandomAccessFile(file, readMode);
}
this.separator = separator;
this.enCycle = enCycle;
this.setLineCount(file);
} catch (FileNotFoundException e) {
e.printStackTrace();
}
}
static Test_IO afc = new Test_IO("F:\\test-io\\random.txt");
public static void main(String[] args) throws InterruptedException {
CountDownLatch countDownLatch = new CountDownLatch(10);
long x = System.currentTimeMillis();
for(int i=0; i<10; i++){
new Thread(new Runnable() {
@Override
public void run() {
while (true) {
String s = afc.readLine();
// System.out.println("==>" + s);
if(s == null) {
break;
}
}
countDownLatch.countDown();
}
},"thread" + i).start();
}
countDownLatch.await();
long y = System.currentTimeMillis();
System.out.println("耗时:" + (y-x));
}
}