java 大文件_Java 超大文件处理

public void parallelReadFile(final String path) throws IOException {

/********************************************************************

* calculate partitions size to map file

********************************************************************/

long K_BYTE = 1024L; // 1KB

long M_BYTE = 1024 * 1024L; // 1MB

long defaultPartitionSize = 50 * M_BYTE;

List partitions = new ArrayList<>();

RandomAccessFile fra = new RandomAccessFile(path, "r");

FileChannel fc_fra = fra.getChannel();

long fileSize = fc_fra.size();

long offset = 0;

while ((offset + defaultPartitionSize) < fileSize) {

offset += defaultPartitionSize;

fc_fra.position(offset);

Scanner scanner = new Scanner(fc_fra);

String line = scanner.nextLine();

partitions.add(defaultPartitionSize + line.length());

offset += line.length();

}

partitions.add(fileSize - offset);

fc_fra.close();

fra.close();

/********************************************************************

* map file & parse in threads

********************************************************************/

ExecutorService threadPool = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());

FileInputStream fis = new FileInputStream(path);

FileChannel fc_fis = fis.getChannel();

offset = 0;

for (Long size : partitions) {

if (size > defaultPartitionSize * 2) {

continue;

}

final long areaFrom = offset;

final long areaTo = offset + size;

final MappedByteBuffer byteBuffer = fc_fis.map(FileChannel.MapMode.READ_ONLY, offset, size);

offset += size;

threadPool.execute(() -> {

try {

Charset charset = Charset.forName("iso-8859-1");

CharsetDecoder decoder = charset.newDecoder();

CharBuffer charBuffer = decoder.decode(byteBuffer);

Scanner scanner = new Scanner(charBuffer);

while (scanner.hasNextLine()) {

String line = scanner.nextLine();

//TODO: parse `line`

}

} catch (CharacterCodingException e) {

logger.error(e.getMessage(), e);

}

});

}

/********************************************************************

* wait until thread pool terminaled

********************************************************************/

threadPool.shutdown();

try {

while (!threadPool.isTerminated()) {

threadPool.awaitTermination(Long.MAX_VALUE, TimeUnit.SECONDS);

}

} catch (InterruptedException e) {

logger.error(e.getMessage(), e);

}

fc_fis.close();

fis.close();

}