高效读取大数据量excle2007文件的解决方案(1)

this.procedure = procedure;

return this;

}

public void endDocument ()

throws SAXException

{

procedure.run();

}

}

package com.apex.bss.mod.util.excle;

/**

• Created by Feng

*/

public interface Procedure {

void run();

}

我们扩展了XSSFSheetXMLHandler类，并且重写了endDocument方法，在原本的类中，该方法没有任何操作，在扩展类中，我们调用Procedure.run方法来处理excle读取完毕时的业务逻辑。

poi自带的事件处理接口是XSSFSheetXMLHandler.SheetContentsHandler，代码如下

public interface SheetContentsHandler {

void startRow(int var1);

void endRow(int var1);

void cell(String var1, String var2, XSSFComment var3);

void headerFooter(String var1, boolean var2, String var3);

}

可以看到数据是基于cell被处理的，我们要在startRow和endRow方法中自己构建行数据

public class SheetHandler implements XSSFSheetXMLHandler.SheetContentsHandler,Procedure {

protected List row = new LinkedList<>();

private ExcleHandler excleHandler;

private int preIndex = 1;

public SheetHandler(ExcleHandler excleHandler){

this.excleHandler = excleHandler;

}

@Override

public void run() {

excleHandler.over();

}

@Override

public void startRow(int i) {

row.clear();

}

@Override

public void endRow(int i) {

excleHandler.handleRow(row,i);

}

@Override

public void cell(String cellReference, String formattedValue, XSSFComment comment) {

int index = excleCelNumToIndex(cellReference);

if(index - preIndex > 1){

for(int i = 0;i < index - preIndex - 1;i++){

row.add(“”);

}

}

row.add(formattedValue);

preIndex = index;

}

@Override

public void headerFooter(String s, boolean b, String s1) {

}

private int excleCelNumToIndex(String celNum){

int num = 0;

int result = 0;

celNum = Pattern.compile(“[\d]”).matcher(celNum).replaceAll(“”);

int length = celNum.length();

for(int i = 0; i < length; i++) {

char ch = celNum.charAt(length - i - 1);

num = (int)(ch - ‘A’ + 1) ;

num *= Math.pow(26, i);

result += num;

}

return result;

}

}

实现了一个事件处理接口，将数据封装成行数据，并且将具体的处理逻辑委托给ExcleHandler接口，此接口基于行来处理excle数据

public interface ExcleHandler {

public void handleRow(Object row, int rowNumber);

public void over();

}

需要注意的是，poi提供给我们的读取工具会跳过某些没有编辑过的单元格，我们需要在构建行的时候自己补全缺失的数据

然后自定义一个ExcleReader用于读取excle

package com.apex.bss.mod.util.excle;

import org.apache.poi.openxml4j.exceptions.InvalidFormatException;

import org.apache.poi.openxml4j.opc.OPCPackage;

import org.apache.poi.openxml4j.opc.PackageAccess;

import org.apache.poi.ss.usermodel.DataFormatter;

import org.apache.poi.util.SAXHelper;

import org.apache.poi.xssf.eventusermodel.ReadOnlySharedStringsTable;

import org.apache.poi.xssf.eventusermodel.XSSFReader;

import org.apache.poi.xssf.eventusermodel.XSSFSheetXMLHandler;

import org.apache.poi.xssf.model.CommentsTable;

import org.apache.poi.xssf.model.StylesTable;

import org.apache.poi.xssf.usermodel.XSSFComment;

import org.xml.sax.InputSource;

import org.xml.sax.SAXException;

import org.xml.sax.XMLReader;

import javax.xml.parsers.ParserConfigurationException;

import java.io.IOException;

import java.io.InputStream;

import java.util.LinkedList;

import java.util.List;

import java.util.regex.Pattern;

/**

• Created by Feng

*/

public class EventExcleReader {

private OPCPackage opcPackage;

private SheetHandler handler;

private ExcleTypeEnum excleTypeEnum;

public EventExcleReader(String filename, ExcleTypeEnum excleType, ExcleHandler handler) throws IllegalAccessException, InvalidFormatException {

if(excleType == ExcleTypeEnum.XLS){

throw new IllegalAccessException(“暂不支持XLS文件”);

}

opcPackage = OPCPackage.open(filename, PackageAccess.READ);

this.handler = new SheetHandler(handler);

}

public EventExcleReader(InputStream inputStream, ExcleTypeEnum excleType, ExcleHandler handler) throws IllegalAccessException, InvalidFormatException, IOException {

if(excleType == ExcleTypeEnum.XLS){

throw new IllegalAccessException(“暂不支持XLS文件”);

}

opcPackage = OPCPackage.open(inputStream);

this.handler = new SheetHandler(handler);

}

public void read(){

InputStream sheetInputStream = null;

try {

XSSFReader xssfReader = new XSSFReader(opcPackage);

StylesTable styles = xssfReader.getStylesTable();

ReadOnlySharedStringsTable strings = new ReadOnlySharedStringsTable(opcPackage);

sheetInputStream = xssfReader.getSheetsData().next();

processSheet(styles, strings, sheetInputStream);

} catch (Exception e) {

throw new RuntimeException(e.getMessage(), e);

}finally {

if(sheetInputStream != null){

try {

sheetInputStream.close();

} catch (IOException e) {

throw new RuntimeException(e.getMessage(), e);

}

}

if(opcPackage != null){

try {

opcPackage.close();

} catch (IOException e) {

throw new RuntimeException(e.getMessage(), e);

}

}

}

}

private void processSheet(StylesTable styles, ReadOnlySharedStringsTable strings, InputStream sheetInputStream) throws SAXException, ParserConfigurationException, IOException{

XMLReader sheetParser = SAXHelper.newXMLReader();

sheetParser.setContentHandler(new LocalXSSFSheetXmlHandler(styles, strings, handler, false).handleEnd(handler));

sheetParser.parse(new InputSource(sheetInputStream));

}

}

然后根据具体业务实现ExcleHandler接口处理业务逻辑即可，这里我们还可以进一步对其进行封装实现数据的分段处理

public class ExcleUtil {

public static void readExcleByReduce(InputStream inputStream, int reduce, ExcleTypeEnum excleType, BiConsumer<LinkedList<LinkedList>,LinkedList> consumer) throws IllegalAccessException, IOException, InvalidFormatException {

new EventExcleReader(inputStream, excleType, new ReduceExcleHandler(consumer).reduceBy(reduce)).read();

}

public static void readExcle(InputStream inputStream,ExcleTypeEnum excleType, BiConsumer<LinkedList<LinkedList>,LinkedList> consumer) throws IllegalAccessException, IOException, InvalidFormatException {

new EventExcleReader(inputStream, excleType, new ReduceExcleHandler(consumer)).read();

}

/**

• 分段读取类

*/

private static class ReduceExcleHandler implements ExcleHandler{

private int reduce = 1;

BiConsumer<LinkedList<LinkedList>,LinkedList> consumer;

private LinkedList<LinkedList> datas = new LinkedList<>();

private LinkedList head;

public ReduceExcleHandler(BiConsumer<LinkedList<LinkedList>,LinkedList> consumer){

this.consumer = consumer;

}

private ReduceExcleHandler reduceBy(int reduce){

this.reduce = reduce;

return this;

}

@Override

public void handleRow(Object row, int rowNumber) {

if(rowNumber <= 0){

head = new LinkedList<>((LinkedList) row);

}else {

if (datas.size() >= reduce) {

consumer.accept(datas, head);

datas.clear();

}

datas.add(new LinkedList<>((LinkedList) row));

}

}

@Override

public void over() {

consumer.accept(datas,head);

}

}

}

既有适合小白学习的零基础资料，也有适合3年以上经验的小伙伴深入学习提升的进阶课程，涵盖了95%以上大数据知识点，真正体系化！

由于文件比较多，这里只是将部分目录截图出来，全套包含大厂面经、学习笔记、源码讲义、实战项目、大纲路线、讲解视频，并且后续会持续更新

需要这份系统化资料的朋友，可以戳这里获取

datas.clear();

}

datas.add(new LinkedList<>((LinkedList) row));

}

}

@Override

public void over() {

consumer.accept(datas,head);

}

}

}

[外链图片转存中…(img-XAa7bQtD-1714208720600)]
[外链图片转存中…(img-fiG6Pcnt-1714208720602)]
[外链图片转存中…(img-YxxvrVhu-1714208720603)]

既有适合小白学习的零基础资料，也有适合3年以上经验的小伙伴深入学习提升的进阶课程，涵盖了95%以上大数据知识点，真正体系化！

由于文件比较多，这里只是将部分目录截图出来，全套包含大厂面经、学习笔记、源码讲义、实战项目、大纲路线、讲解视频，并且后续会持续更新

需要这份系统化资料的朋友，可以戳这里获取