package com.my.tpp.utils;
import cn.hutool.core.codec.Base64Encoder;
import cn.hutool.core.date.DateTime;
import com.github.pagehelper.util.StringUtil;
import lombok.extern.slf4j.Slf4j;
import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import lombok.extern.slf4j.Slf4j;
import net.sf.sevenzipjbinding.*;
import net.sf.sevenzipjbinding.impl.RandomAccessFileInStream;
import okio.BufferedSink;
import okio.BufferedSource;
import okio.Okio;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.poi.EncryptedDocumentException;
import org.apache.poi.POIXMLDocumentPart;
import org.apache.poi.hssf.usermodel.*;
import org.apache.poi.openxml4j.exceptions.InvalidFormatException;
import org.apache.poi.ss.usermodel.*;
import org.apache.poi.xssf.usermodel.*;
import org.openxmlformats.schemas.drawingml.x2006.spreadsheetDrawing.CTMarker;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.mock.web.MockMultipartFile;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import org.springframework.web.multipart.MultipartFile;
import java.io.*;
import java.math.BigDecimal;
import java.util.*;
import java.nio.file.Path;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.List;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.ReentrantLock;
import java.util.stream.Collectors;
import java.util.zip.ZipEntry;
import java.util.zip.ZipException;
import java.util.zip.ZipFile;
import java.util.zip.ZipInputStream;
/**
* @ClassName PicturesVo
* @Description TODO
* @Author fan
* @Date 2023/5/22 9:33
* @Version 1.0
*/
@Slf4j
@Service
public class 你的Util {
private static final String format = "yyyy/MM/dd HH:mm:ss";
private static SimpleDateFormat sdf = new SimpleDateFormat(format);
public boolean checkWord(String word) {
return !StringUtil.isEmpty(word);
}
/**
* @Description:获取excel表中的图片
* @throws IOException
* @throws EncryptedDocumentException
* @param file 文件输入流
* @param sheetNum Excel表中的sheet编号
* @return: java.util.Map<java.lang.String,org.apache.poi.ss.usermodel.PictureData>
* @Author: fan
* @Date: 2023/5/22 11:05
*/
public static Map<String, PictureData> getPictureFromExcel(File file, int sheetNum) throws EncryptedDocumentException, IOException {
//获取图片PictureData集合
String fileName = file.getName();
Workbook workbook = null;
FileInputStream fileInputStream = null;
try {
if (StringUtil.isEmpty(fileName)) {
return null;
}
fileInputStream = new FileInputStream(file);
if (fileName.endsWith("xls")) {
//2003
workbook = new HSSFWorkbook(fileInputStream);
HSSFSheet sheet = (HSSFSheet) workbook.getSheetAt(sheetNum - 1);
Map<String, PictureData> pictures = getPictures(sheet);
return pictures;
} else if (fileName.endsWith("xlsx")) {
//2007
workbook = new XSSFWorkbook(fileInputStream);
XSSFSheet sheet = (XSSFSheet) workbook.getSheetAt(sheetNum - 1);
Map<String, PictureData> pictures = getPictures(sheet);
return pictures;
}
} catch (Exception e) {
log.error("[getPictureFromExcel()-Exception] Exception状态异常 --->:{} , 栈信息:{}" , e.getMessage() , getExceptionStr(e));
} finally {
try {
if (fileInputStream != null) {
fileInputStream.close();
}
} catch (Exception ee) {
log.error("[getPictureFromExcel()-Exception] Exception状态异常2 --->:{} , " , ee.getMessage() );
}
}
return new HashMap();
}
/**
* @Description:获取excel表中的图片
* @throws IOException
* @throws EncryptedDocumentException
* @param multipartFile 文件输入流
* @param sheetNum Excel表中的sheet编号
* @return: java.util.Map<java.lang.String,org.apache.poi.ss.usermodel.PictureData>
* @Author: fan
* @Date: 2023/5/22 11:05
*/
public static Map<String, PictureData> getPictureFromExcel(MultipartFile multipartFile, int sheetNum) {
//获取图片PictureData集合
Workbook workbook = null;
InputStream inputStream = null;
try {
if (multipartFile != null) {
inputStream = multipartFile.getInputStream();
workbook = new XSSFWorkbook(inputStream);
XSSFSheet sheet = (XSSFSheet) workbook.getSheetAt(sheetNum - 1);
Map<String, PictureData> pictures = getPictures(sheet);
return pictures;
}
} catch (Exception e) {
log.error("[getAlertInfoList()-Exception] Exception状态异常 --->:{}" , e.getMessage() );
} finally {
try {
if (inputStream != null) {
inputStream.close();
}
} catch (Exception e1) {
log.error("[getAlertInfoList()-Exception] Exception状态异常1 --->:{} , " , e1.getMessage() );
}
}
return new HashMap();
}
/**
* 获取图片和位置 (xls版)
* @param sheet
* @return
* @throws IOException
*/
public static Map<String, PictureData> getPictures (HSSFSheet sheet) {
Map<String, PictureData> map = new HashMap<String, PictureData>();
List<HSSFShape> list = sheet.getDrawingPatriarch().getChildren();
for (HSSFShape shape : list) {
if (shape instanceof HSSFPicture) {
HSSFPicture picture = (HSSFPicture) shape;
HSSFClientAnchor cAnchor = picture.getClientAnchor();
PictureData pdata = picture.getPictureData();
String key = cAnchor.getRow1() + "-" + cAnchor.getCol1(); // 行号-列号
map.put(key, pdata);
}
}
return map;
}
/**
* 获取图片和位置 (xlsx版)
* @param sheet
* @return
* @throws IOException
*/
public static Map<String, PictureData> getPictures (XSSFSheet sheet) {
Map<String, PictureData> map = new HashMap<String, PictureData>();
List<POIXMLDocumentPart> list = sheet.getRelations();
for (POIXMLDocumentPart part : list) {
if (part instanceof XSSFDrawing) {
XSSFDrawing drawing = (XSSFDrawing) part;
List<XSSFShape> shapes = drawing.getShapes();
for (XSSFShape shape : shapes) {
/*XSSFPicture picture = (XSSFPicture) shape;
XSSFClientAnchor anchor = picture.getPreferredSize();
CTMarker marker = anchor.getFrom();
String key = marker.getRow() + "-" + marker.getCol();
map.put(key, picture.getPictureData());*/
XSSFPicture picture = (XSSFPicture) shape;
//解决图片空指针报错问题 fan 2023-05-30
XSSFClientAnchor anchor = (XSSFClientAnchor) shape.getAnchor();
// XSSFClientAnchor anchor = pic.getPreferredSize();
CTMarker ctMarker = anchor.getFrom();
//解决用户头像格式错误 fan 2023-05-30
String picIndex = ctMarker.getRow() + "-" + anchor.getCol1();
map.put(picIndex, picture.getPictureData());
}
}
}
return map;
}
public static String doZero(String value) {
String val = value;
if (value.indexOf(".") != -1) {
val = value.replaceAll("0+?$", "");
val = val.replaceAll("[.]$", "");
}
return val;
}
/**
* @desc:获取单元格内容
* @param cell
* @return
* 异常时返回null
*/
public static String getCellValueText(Cell cell) {
try {
String text = "";
if (cell == null) {
return text;
} else {
switch (cell.getCellType()) {
case Cell.CELL_TYPE_NUMERIC://0
if (DateUtil.isCellDateFormatted(cell)) {// 判断是否为日期类型
Date date = DateUtil.getJavaDate(cell.getNumericCellValue());
return DateUtils.covertDate2String(date);//这里做个时间转换就补贴了
}
Double value = cell.getNumericCellValue();
BigDecimal bd = new BigDecimal(value.toString());
text = bd.toPlainString();
text = doZero(text);
break;
case Cell.CELL_TYPE_STRING://1
text = cell.getStringCellValue().trim();
case Cell.CELL_TYPE_FORMULA://2
try {
text = String.valueOf(cell.getNumericCellValue());
} catch (IllegalStateException e) {
text = String.valueOf(cell.getRichStringCellValue());
}
break;
}
return text;
}
} catch (Exception e) {
log.error("[getCellValueText()-Exception] Exception状态异常 --->:{} " , e.getMessage() );
return null;
}
}
/**
* @Description:获取excel里面的内容
* @param multipartFile
* @return: java.util.List<java.util.Map<java.lang.String,java.lang.Object>>
* @Author: fan
* @Date: 2023/6/14 11:02
*/
public static List<Map<String,Object>> getPictureList(MultipartFile multipartFile){
List<Map<String,Object>> list = new ArrayList<>();
try {
//获取图片数据
Map<String, PictureData> map = getPictureFromExcel(multipartFile, 1);
InputStream inputStream = multipartFile.getInputStream();
Workbook workbook = WorkbookFactory.create(inputStream);
Sheet sheet = workbook.getSheetAt(0);
int lastRow = sheet.getLastRowNum();
int lastCell = sheet.getRow(0).getLastCellNum();
for (int i = 1; i <= lastRow; i++) {
Row row = sheet.getRow(i);
if (row == null) {
continue;
}
Map<String, Object> param = new HashMap<>();
for (int j = 0; j < lastCell; j++) {
Cell cell = row.getCell(j);
String value = getCellValueText(cell);
if (value != null) {
value = value.trim();
}
if (j == 0) {
param.put("你的用户名", value);
}
if (j == 1) {
param.put("你的备注", value);
}
if (j == 2) {
param.put("你的类型", value);
}
if (j == 3) {
//根据图片行列位置获取图片
String key = "" + i + "-" + j;
PictureData picData = map.get(key);
if (picData != null) {
byte[] data = picData.getData();
if (data.length > 0) {
String base64 = Base64Encoder.encode(data);
param.put("你的图", base64);
/*String extension = picData.suggestFileExtension();
FileOutputStream out = new FileOutputStream("C:\\my-word\\picture\\" + key + "_picture." + extension);
out.write(data);
out.close();*/
}
}
}
}
if (!CollectionUtils.isEmpty(param)) {
list.add(param);
}
}
} catch (Exception e){
log.error("[getPictureList2()-Exception] Exception状态异常2 --->:{} " , e.getMessage() );
} finally {
try {
if (multipartFile.getInputStream() != null) {
multipartFile.getInputStream().close();
}
} catch (Exception ee) {
log.error("[getPictureList1()-Exception] Exception状态异常1 --->:{}" , ee.getMessage());
}
}
return list;
}
/**
* @Description: file转byte[]
* @param file
* @return: byte[]
* @Author: fan
* @Date: 2023/6/21 10:09
*/
public static byte[] fileConvert2ByteArray(File file) {
byte[] data = null;
try {
FileInputStream fis = new FileInputStream(file);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
int len;
byte[] buffer = new byte[1024];
while ((len = fis.read(buffer)) != -1) {
baos.write(buffer, 0, len);
}
data = baos.toByteArray();
fis.close();
baos.close();
} catch (Exception e) {
log.error("[fileConvert2ByteArray()--关闭流2] Message={}",e.getMessage());
}
return data;
}
public static Workbook getWorkbook(MultipartFile multipartFile) throws IOException, InvalidFormatException {
Workbook workbook = WorkbookFactory.create(multipartFile.getInputStream());
return workbook;
}
private static final int rowNum = 20;
/**
* @Description:获取excel的内容
* @param multipartFile
* @return: java.util.List<java.util.Map<java.lang.String,java.lang.Object>>
* @Author: fan
* @Date: 2023/6/14 14:31
*/
public List<Map<String,Object>> getEXcelList(MultipartFile multipartFile) throws IOException, InvalidFormatException, InterruptedException, ExecutionException {
List<Map<String,Object>> list = new ArrayList<>();
//try {
//获取图片数据
Workbook workbook = getWorkbook(multipartFile);
Sheet sheet = workbook.getSheetAt(0);
int lastRow = sheet.getLastRowNum();
log.info("获取到workbook中的最后行数 lastRow={}", lastRow);
if (lastRow > 6 && lastRow <= 101) {
int allNumberOfRows = sheet.getPhysicalNumberOfRows();
log.info("获取到workbook中的总行数:{}", allNumberOfRows);
int rows = allNumberOfRows - 1;
int threadNum = rows / rowNum + 1;
//设置一个信号量为10的信号量,限制同时运行的线程数量最大为10
Semaphore semaphore = new Semaphore(10);
CountDownLatch countDownLatch = new CountDownLatch(threadNum);
ExecutorService executor = Executors.newFixedThreadPool(threadNum);
log.info("开始创建线程,数据总行数:{},线程数量:{}", rows, threadNum);
List<Future<List<Map<String, Object>>>> futures = new ArrayList<>();
for (int i = 1; i <= threadNum; i++) {
int startRow = (i - 1) * rowNum + 1;
int endRow = i * rowNum;
if (i == threadNum) {
endRow = rows;
}
log.info("开始执行线程方法,线程ID:<{}>,线程名称:<{}>", Thread.currentThread().getId(), Thread.currentThread().getName());
Future<List<Map<String, Object>>> future = executor.submit(
new ExcelTask(semaphore, workbook, startRow, endRow, countDownLatch,this));
futures.add(future);
}
for (Future<List<Map<String, Object>>> future : futures) {
list.addAll(future.get());
}
countDownLatch.await();
executor.shutdown();
}
if (lastRow > 0 && lastRow <= 6) {
StringBuilder sb = new StringBuilder();
int lastCell = sheet.getRow(0).getLastCellNum();
for (int i = 1; i <= lastRow; i++) {
Row row = sheet.getRow(i);
if (row == null) {
continue;
}
Map<String, Object> param = new HashMap<>();
for (int j = 0; j < lastCell; j++) {
Cell cell = row.getCell(j);
String value = getCellValueText(cell);
if (value != null) {
value = value.trim();
}
if (j == 0) {
if (StringUtils.isEmpty(value)) {
}
if (!StringUtil.isEmpty(value) && value.length() > 30) {
}
sb.append(value).append("_").append(i).append(".").append(j).append("|");
param.put("你的姓名", value);
}
if (j == 1) {
if (StringUtils.isEmpty(value)) {
}
if (!StringUtil.isEmpty(value) && value.length() > 30) {
}
sb.append(value).append("_").append(i).append(".").append(j).append("|");
param.put("你的备注", value);
}
if (j == 2) {
if (StringUtils.isEmpty(value)) {
}
if("你的黑名单".equals(value)){
}else if("你的白名单".equals(value)){
}else {
log.info("batchAddFace type error");
}
param.put("你的类型", value);
}
}
if (checkWord(sb.toString())) {
}
if (!CollectionUtils.isEmpty(param)) {
list.add(param);
}
}
}
multipartFile.getInputStream().close();
return list;
}
/**
* @Description:批量上传文件,从excel文件和图片压缩包整合数据
* @param files
* @return: java.util.List<java.util.Map<java.lang.String,java.lang.Object>>
* @Author: fan
* @Date: 2023/6/14 15:14
*/
public List<Map<String,Object>> getPictureXlsxList(MultipartFile[] files) throws Exception{
long startTime_0 = System.currentTimeMillis();
log.info("[getPictureXlsxList()--开始执行] startTime_0={}",startTime_0);
if (files == null) {
throw new BaseBackRuntimeException(ResultEnum.FILE_EXIST);
}
long count = Arrays.stream(files).count();
if (count > 2) {
throw new BaseBackRuntimeException(ResultEnum.FILE_COUNT);
}
List<Map<String, Object>> list = new ArrayList<>();
List<Map<String, Object>> eXcelList = new ArrayList<>();
Map<String, Object> param = new HashMap<>();
for (MultipartFile multipartFile : files) {
String originalFilename = multipartFile.getOriginalFilename();
String fileSuffixName = originalFilename.substring(originalFilename.lastIndexOf(".") + 1);
log.info("[getPictureXlsxList()--上传的后缀名] fileSuffixName={} , originalFilename={}",fileSuffixName,originalFilename);
if ("xlsx".equals(fileSuffixName) || "xlsm".equals(fileSuffixName)) {
long startTime = System.currentTimeMillis();
eXcelList = getEXcelList(multipartFile);
long entTime = System.currentTimeMillis();
log.info("[excel线程耗时:time] excel_pool={}", entTime - startTime);
}
if ("zip".equals(fileSuffixName)) {
long startTime_zip = System.currentTimeMillis();
File file = multiPartFile2File(multipartFile);
ZipFile zipFile = new ZipFile(file);
int size = zipFile.size();
//unZipSingleThread(multipartFile, param);
if (size > 0 && size <= 5) {
long startTime = System.currentTimeMillis();
unZipSingleThread(multipartFile, param);
long entTime = System.currentTimeMillis();
log.info("[单线程耗时:time] pool={}", entTime - startTime);
} else if (size > 5) {
long startTime = System.currentTimeMillis();
//unZipMoreThread(multipartFile, null , param , size);
unZipMoreThread(param,zipFile);
long entTime = System.currentTimeMillis();
log.info("[多线程耗时:time] pool={}", entTime - startTime);
}
if (file != null) {
file.delete();
}
long entTime_zip = System.currentTimeMillis();
log.info("[zip_线程耗时:time] zip_pool={}", entTime_zip - startTime_zip);
}
if ("rar".equals(fileSuffixName)) {
File file = multiPartFile2File(multipartFile);
//unrar(file, file, param);
unMoreThreadRar(file,param);
file.delete();
}
}
if (!CollectionUtils.isEmpty(eXcelList) && !CollectionUtils.isEmpty(param)) {
log.info("[getPictureXlsxList()--执行中...] eXcelList_size={}, param_size={}",eXcelList.size(),param.size());
int size = eXcelList.size();
int total = param.size();
if (size != total) {
}
for (int i = 0; i < eXcelList.size(); i++) {
Map<String, Object> map = eXcelList.get(i);
String name = (String) map.get("你的姓名");
Object value = param.get(name);
if(value != null){
Map<String, Object> mapPic = new HashMap<>();
mapPic.put("你的图", value);
mapPic.put("你的姓名", name);
mapPic.put("你的备注", map.get("你的备注"));
mapPic.put("你的类型", map.get("你的类型"));
list.add(mapPic);
} else {
log.info("[getPictureXlsxList()--不匹配] 第" + i + "行不匹配");
}
}
}
long entTime_0 = System.currentTimeMillis();
log.info("[getPictureXlsxList()--执行成功] entTime_0={}, 执行时间poolTime={}",entTime_0,entTime_0 - startTime_0);
return list;
}
public static void unZipSingleThread(MultipartFile multipartFile,Map<String, Object> param) throws IOException {
InputStream inputStream = multipartFile.getInputStream();
// 这里假设已经通过MultipartFile对象接收了上传的zip文件,存在变量multipartFile中
// 创建ZipInputStream
ZipInputStream zipInputStream = new ZipInputStream(inputStream);
ZipEntry zipEntry = null;
// 循环遍历zip文件中的每一个文件片段
while ((zipEntry = zipInputStream.getNextEntry()) != null) {
if (!zipEntry.isDirectory()) {
// 生成临时文件来保存文件片段
String name = zipEntry.getName();
File tempFile = File.createTempFile(name, null);
FileOutputStream fos = new FileOutputStream(tempFile);
// 读取文件片段,并保存到临时文件中
byte[] buffer = new byte[1024];
int len = 0;
while ((len = zipInputStream.read(buffer)) > 0) {
fos.write(buffer, 0, len);
}
fos.close();
// 处理临时文件,例如将其存储到数据库中
// ...
byte[] bytes = fileConvert2ByteArray(tempFile);
String value = Base64Encoder.encode(bytes);
String key = name.substring(0, name.lastIndexOf("."));
param.put(key,value);
// 删除临时文件
tempFile.delete();
}
}
zipInputStream.close();
inputStream.close();
}
public static List<File> unrar(File file, File targetDir , Map<String , Object> map) {
List<File> targetFiles = new ArrayList<>();
RandomAccessFile randomAccessFile = null;
IInArchive archive = null;
try {
if(!targetDir.exists()){
targetDir.mkdirs();
}
randomAccessFile = new RandomAccessFile(file, "rw");
RandomAccessFileInStream randomAccessFileInStream = new RandomAccessFileInStream(randomAccessFile);
archive = SevenZip.openInArchive(null,randomAccessFileInStream);
int[] arr = new int[archive.getNumberOfItems()];
for (int i = 0; i < arr.length; i++) {
arr[i] = i;
}
archive.extract(arr, false, getExtractCallback(archive,targetDir,targetFiles,map));
} catch (Exception e) {
log.error("[unrar()--异常1] Message={}, e={}",e.getMessage(),getExceptionStr(e));
} finally {
if (archive != null) {
try {
archive.close();
} catch (SevenZipException e1) {
log.error("[unrar()--异常2] Message={}, e1={}",e1.getMessage(),getExceptionStr(e1));
}
}
if (randomAccessFile != null) {
try {
randomAccessFile.close();
} catch (IOException e1) {
log.error("[unrar()--异常3] Message={}, e={}",e1.getMessage(),getExceptionStr(e1));
}
}
}
return targetFiles;
}
public Map<String, Object> unMoreThreadRar(File file,Map<String, Object> param){
long startTime_0 = System.currentTimeMillis();
Map<String, Object> map = new HashMap<>();
RandomAccessFile randomAccessFile = null;
IInArchive archive = null;
int threadNum = Runtime.getRuntime().availableProcessors();
if (threadNum <= 2) {
threadNum = 8;
}
log.info("线程数threadNum-->" + threadNum);
CountDownLatch countDownLatch = new CountDownLatch(threadNum);
ExecutorService threadPool = Executors.newFixedThreadPool(threadNum);
try {
if(!file.exists()){
file.mkdirs();
}
randomAccessFile = new RandomAccessFile(file, "rw");
RandomAccessFileInStream randomAccessFileInStream = new RandomAccessFileInStream(randomAccessFile);
archive = SevenZip.openInArchive(null,randomAccessFileInStream);
Future<Map<String, Object>> future = threadPool.submit(
new UnRarFileTask(randomAccessFile, archive, param, file, countDownLatch));
map = future.get();
countDownLatch.await(2,TimeUnit.MICROSECONDS);
threadPool.shutdown();
long entTime_0 = System.currentTimeMillis();
log.info("[unMoreThreadRar()--执行耗时:] checkFile_pool_time={}",entTime_0 - startTime_0);
//threadPool.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS);
} catch (Exception e) {
log.error("[unrar()--异常1] Message={}, e={}",e.getMessage(),getExceptionStr(e));
} finally {
try {
if (archive != null) {
archive.close();
}
if (randomAccessFile != null) {
randomAccessFile.close();
}
} catch (Exception e1) {
log.error("[unrar()--异常2] Message={}, e1={}",e1.getMessage(),getExceptionStr(e1));
}
return map;
}
}
private static class UnRarTask implements Runnable {
private IInArchive archive;
private int[] arr;
private Map<String, Object> param;
private File file;
private CountDownLatch countDownLatch;
private RandomAccessFile randomAccessFile;
public UnRarTask(IInArchive archive, int[] arr, Map<String, Object> param, File file,CountDownLatch countDownLatch) {
this.archive = archive;
this.arr = arr;
this.param = param;
this.file = file;
this.countDownLatch = countDownLatch;
}
public UnRarTask(RandomAccessFile randomAccessFile,Map<String, Object> param,File file,CountDownLatch countDownLatch) {
this.randomAccessFile = randomAccessFile;
this.param = param;
this.file = file;
this.countDownLatch = countDownLatch;
}
public UnRarTask(Map<String, Object> param,File file,CountDownLatch countDownLatch) {
this.param = param;
this.file = file;
this.countDownLatch = countDownLatch;
}
@Override
public void run() {
try {
countDownLatch.countDown();
archive.extract(arr, false, getExtractCallback(archive,file,null,param));
} catch (SevenZipException e) {
throw new RuntimeException(e);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
}
public static List<Map<String,Object>> getPictureList(String filePath){
List<Map<String,Object>> list = new ArrayList<>();
FileInputStream fileInputStream = null;
MultipartFile multipartFile = null;
try {
File file = new File(filePath);
//获取图片数据
Map<String, PictureData> map = getPictureFromExcel(file, 1);
fileInputStream = new FileInputStream(filePath);
multipartFile = new MockMultipartFile("test", fileInputStream);
Workbook workbook = WorkbookFactory.create(multipartFile.getInputStream());
Sheet sheet = workbook.getSheetAt(0);
int lastRow = sheet.getLastRowNum();
int lastCell = sheet.getRow(0).getLastCellNum();
System.out.println(lastRow + ", " + lastCell);
for (int i = 1; i <= lastRow; i++) {
Row row = sheet.getRow(i);
if (row == null) {
continue;
}
Map<String, Object> param = new HashMap<>();
for (int j = 0; j < lastCell; j++) {
Cell cell = row.getCell(j);
String value = getCellValueText(cell);
if (value != null) {
value = value.trim();
}
if (j == 0) {
param.put("你的用户名", value);
}
if (j == 1) {
param.put("你的备注", value);
}
if (j == 2) {
param.put("你的类型", value);
}
if (j == 3) {
//根据图片行列位置获取图片
String key = "" + i + "-" + j;
PictureData picData = map.get(key);
if (picData != null) {
byte[] data = picData.getData();
if (data.length > 0) {
String base64 = Base64Encoder.encode(data);
param.put("你的图", base64);
/*String extension = picData.suggestFileExtension();
FileOutputStream out = new FileOutputStream("C:\\my-word\\picture\\" + key + "_picture." + extension);
out.write(data);
out.close();*/
}
}
}
}
if (!CollectionUtils.isEmpty(param)) {
list.add(param);
}
}
} catch (Exception e){
log.error("[getPictureList()-Exception] Exception状态异常 --->:{}" , e.getMessage() );
} finally {
try {
if (multipartFile.getInputStream() != null) {
multipartFile.getInputStream().close();
}
if (fileInputStream != null){
fileInputStream.close();
}
} catch (Exception ee){
log.error("[getPictureList()-Exception] Exception状态异常 --->:{} " , ee.getMessage() );
}
}
return list;
}
private static IArchiveExtractCallback getExtractCallback(IInArchive inArchive, File targetDir, List<File> targetFiles , Map<String , Object> map){
return new IArchiveExtractCallback(){
@Override
public void setTotal(long l) throws SevenZipException {}
@Override
public void setCompleted(long l) throws SevenZipException {}
@Override
public ISequentialOutStream getStream(int index, ExtractAskMode extractAskMode) throws SevenZipException {
final String fileName = (String) inArchive.getProperty(index, PropID.PATH);
log.info("index-->" + index + ", fileName-->" + fileName);
//final boolean isFolder = (boolean) inArchive.getProperty(index, PropID.IS_FOLDER);
return data -> {
//String value = Base64Encoder.encode(data);
String key = fileName.substring(0, fileName.lastIndexOf("."));
byte[] bytes = fileConvert2ByteArray(targetDir);
String value = Base64Encoder.encode(bytes);
if (map.get(key) == null) {
map.put(key, value);
}
return data.length;
};
}
@Override
public void prepareOperation(ExtractAskMode extractAskMode) throws SevenZipException {}
@Override
public void setOperationResult(ExtractOperationResult extractOperationResult) throws SevenZipException {}
};
}
/**
* @Description: MultipartFile转File
* @param multipartFile
* @return: java.io.File
* @Author: fan
* @Date: 2023/6/20 14:30
*/
public static File multiPartFileToFile(MultipartFile multipartFile) throws IOException {
//获取文件名
String originalFilename = multipartFile.getOriginalFilename();
//获取默认定位到的当前用户目录("user.dir"),也就是当前应用的根路径
String tempDir = System.getProperty("user.dir");
//根目录下生成临时文件
File file = new File(tempDir+File.separator+originalFilename);
FileUtils.copyInputStreamToFile(multipartFile.getInputStream(), file);
return file;
}
/**
* @Description: MultipartFile转File
* @param multipartFile
* @return: java.io.File
* @Author: fan
* @Date: 2023/6/20 14:30
*/
public static File multiPartFile2File(MultipartFile multipartFile) throws IOException {
//获取文件名
String originalFilename = multipartFile.getOriginalFilename();
//获取默认定位到的当前用户目录("user.dir"),也就是当前应用的根路径
String tempDir = System.getProperty("user.dir");
//获取文件名
String filename = StringUtils.substringBeforeLast(originalFilename, ".");
//获取文件后缀
String fileExt = StringUtils.substringAfterLast(originalFilename, ".");
//在项目根路径生成临时文件
File tempFile = File.createTempFile(filename, "." + fileExt, new File(tempDir));
multipartFile.transferTo(tempFile);
//在jvm退出时执行删除此路径下的文件
tempFile.deleteOnExit();
return tempFile;
}
/**
复制文件夹
@param file 原文件夹
@param file1 复制后的文件夹
*/
public static void copyDir(File file, File file1) {
if (!file.isDirectory()) {
return;
}
if (!file1.exists()) {
file1.mkdirs();
}
File[] files = file.listFiles();
for (int i = 0; i < files.length; i++) {
File f = files[i];
if (f.isDirectory()) {
copyDir(f, new File(file1.getPath(), f.getName()));
} else if (f.isFile()) {
copyFile(f, new File(file1.getPath(), f.getName()));
}
}
}
/**
复制文件
@param file 原文件
@param file1 复制后的文件
*/
public static void copyFile(File file, File file1) {
try (FileInputStream fis = new FileInputStream(file); FileOutputStream fos = new FileOutputStream(file1)) {
byte[] bys = new byte[1024];
int len;
while ((len = fis.read(bys)) != -1) {
fos.write(bys, 0, len);
}
} catch (Exception e) {
log.error("[copyFile()--异常] Message={}",e.getMessage());
}
}
private static final int THREAD_POOL_SIZE = 5;
public static void unZipMoreThread(MultipartFile file, String destinationPath, Map<String, Object> param , Integer size) throws IOException, InterruptedException {
/*Path destDir = Paths.get(destinationPath);
if (!Files.exists(destDir)) {
Files.createDirectories(destDir);
}*/
InputStream inputStream = file.getInputStream();
ZipInputStream zipInputStream = new ZipInputStream(inputStream);
int rows = size - 1;
int threadNum = rows / rowNum + 1;
ExecutorService executorService = Executors.newFixedThreadPool(threadNum * 2);
Semaphore semaphore = new Semaphore(THREAD_POOL_SIZE * 2);
CountDownLatch countDownLatch = new CountDownLatch(threadNum);
ReentrantLock lock = new ReentrantLock();
ZipEntry zipEntry = null;
while ((zipEntry = zipInputStream.getNextEntry()) != null) {
if (!zipEntry.isDirectory()) {
executorService.execute(
new UnzipTask(zipInputStream,
//"null",
zipEntry, param, countDownLatch,semaphore, lock));
try {
Thread.sleep(rowNum + rowNum + 10);
} catch (Exception e) {
log.error("[unZipMoreThread()--异常] Message={}",e.getMessage());
}
}
}
/*int count = param.size();
if (size != count) {
unZipMoreThread( file, destinationPath, param , size);
log.info("[unZipMoreThread()--重试] size={},count={}",size,count);
}*/
countDownLatch.await();
executorService.shutdown();
zipInputStream.closeEntry();
zipInputStream.close();
inputStream.close();
}
private static class UnzipTask implements Runnable {
private ZipInputStream zipInputStream;
private Path destDir;
private ZipEntry entry;
private Map<String, Object> param;
private CountDownLatch countDownLatch;
private Semaphore semaphore;
private ReentrantLock lock;
public UnzipTask(ZipInputStream zipInputStream,
//Path destDir,
ZipEntry entry,
Map<String, Object> param,
CountDownLatch countDownLatch, Semaphore semaphore, ReentrantLock lock) {
this.zipInputStream = zipInputStream;
//this.destDir = destDir;
this.entry = entry;
this.param = param;
this.countDownLatch = countDownLatch;
this.semaphore = semaphore;
this.lock = lock;
}
public void run() {
try {
/*log.info("解压zip线程ID:<{}>开始运行",Thread.currentThread().getId());
semaphore.acquire();
log.info("解压zip消耗了一个信号量,剩余信号量为:{}",semaphore.availablePermits());*/
countDownLatch.countDown();
/*if (destDir != null) {
Path filePath = destDir.resolve(entryName);
Files.createDirectories(filePath.getParent());
BufferedOutputStream outputStream =
new BufferedOutputStream(
new FileOutputStream(
filePath.toFile()), 1024);
byte[] buffer = new byte[1024];
int length = 0;
while ((length = zipInputStream.read(buffer, 0, 1024)) != -1) {
outputStream.write(buffer, 0, length);
}
outputStream.flush();
outputStream.close();
} else {*/
/*ZipEntry entry;
while ((entry = zipInputStream.getNextEntry()) != null) {
if (!entry.isDirectory()) {*/
String entryName = entry.getName();
log.info("解压zip里文件名称:{}",entryName);
//lock.lock();
File tempFile = File.createTempFile(entryName, null);
FileOutputStream fos = new FileOutputStream(tempFile);
// 读取文件片段,并保存到临时文件中
byte[] buffer = new byte[1024];
int len = 0;
while ((len = zipInputStream.read(buffer)) > 0) {
fos.write(buffer, 0, len);
}
fos.close();
// 处理临时文件,例如将其存储到数据库中
// ...
byte[] bytes = fileConvert2ByteArray(tempFile);
String value = Base64Encoder.encode(bytes);
String key = entryName.substring(0, entryName.lastIndexOf("."));
param.put(key, value);
// 删除临时文件
tempFile.delete();
//lock.unlock();
/* }
}*/
//semaphore.release();//释放并将其返回给信号量
} catch (Exception e) {
log.error("[unZipMoreThread()--run()异常] Message={}",e.getMessage());
}
}
}
private ZipFile zipFile;
private String destPath;
/**
* @Description:解压ZIP
* @param map
* @param zipFile
* @Author: fan
* @Date: 2023/7/7 14:49
*/
public void unZipMoreThread(Map<String,Object> map,ZipFile zipFile) throws IOException {
long start = System.currentTimeMillis();
log.info("[unZip()--开始解压] start={}",start);
//this.destPath = destPath;
//this.zipFile = new ZipFile(srcPath);
Enumeration<? extends ZipEntry> entries = null;
int threadNum = Runtime.getRuntime().availableProcessors();
if (threadNum <= 2) {
threadNum = 8;
}
log.info("线程数threadNum-->" + threadNum);
CountDownLatch countDownLatch = new CountDownLatch(threadNum);
ExecutorService threadPool = Executors.newFixedThreadPool(threadNum);
try {
entries = zipFile.entries();
} catch (Exception e) {
log.error("[unZip()--entries解压异常] Message={}",e.getMessage());
}
try {
if (entries != null) {
while (entries.hasMoreElements()) {
ZipEntry zipEntry = entries.nextElement();
if (zipEntry.isDirectory()) {
continue;
}
threadPool.execute(new ZipFileTask(zipEntry, map, countDownLatch,zipFile));
//Thread.sleep(rowNum);
}
log.info("[我顶你的肺]");
} else {
log.info("[1我顶你的肺_2]");
}
countDownLatch.await();
threadPool.shutdown();
threadPool.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS);
} catch (InterruptedException e) {
log.error("[unZip()--解压异常] Message={}, e={}",e.getMessage(),getExceptionStr(e));
}
zipFile.close();
long end = System.currentTimeMillis();
log.info("解压完成,所用时间为:" + (end - start) + "ms");
}
public void decompress(String srcPath, String destPath) throws IOException {
long start = System.currentTimeMillis();
this.destPath = destPath;
this.zipFile = new ZipFile(srcPath);
Enumeration<? extends ZipEntry> entries = zipFile.entries();
ExecutorService threadPool = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
while (entries.hasMoreElements()) {
ZipEntry zipEntry = entries.nextElement();
if (zipEntry.isDirectory()) {
continue;
}
threadPool.execute(new ZipFileTask(zipEntry));
}
threadPool.shutdown();
try {
threadPool.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS);
} catch (InterruptedException e) {
e.printStackTrace();
log.error("[getPictureList()-Exception] Exception状态异常 --->:{} " , e.getMessage() );
}
zipFile.close();
long end = System.currentTimeMillis();
System.out.println("解压完成,所用时间为:" + (end - start) + "ms");
}
private class ZipFileTask implements Runnable {
private ZipEntry zipEntry;
private Map<String, Object> param;
private CountDownLatch countDownLatch;
private ZipFile zipFile;
ZipFileTask(ZipEntry zipEntry) {
this.zipEntry = zipEntry;
}
ZipFileTask(ZipEntry zipEntry,Map<String, Object> param) {
this.zipEntry = zipEntry;
this.param = param;
}
ZipFileTask(ZipEntry zipEntry,Map<String, Object> param,CountDownLatch countDownLatch,ZipFile zipFile) {
this.zipEntry = zipEntry;
this.param = param;
this.countDownLatch = countDownLatch;
this.zipFile = zipFile;
}
@Override
public void run() {
log.info("解压zip线程ID:<{}>开始运行",Thread.currentThread().getId());
countDownLatch.countDown();
File file = null;
//new File(destPath + File.separator + zipEntry.getName());
String name = zipEntry.getName();
/*File parentFile = file.getParentFile();
if (!parentFile.exists()) {
parentFile.mkdirs();
}*/
try /*(
InputStream inputStream = zipFile.getInputStream(this.zipEntry);
OutputStream outputStream = new FileOutputStream(file);
) */{
/*int read;
byte[] bytes = new byte[inputStream.available()];
while ((read = inputStream.read(bytes)) != -1) {
outputStream.write(bytes, 0, read);
}*/
file = File.createTempFile(zipEntry.getName(), null);
InputStream inputStream = zipFile.getInputStream(this.zipEntry);
FileOutputStream fos = new FileOutputStream(file);
// 读取文件片段,并保存到临时文件中
byte[] buffer = new byte[1024];
int len = 0;
while ((len = inputStream.read(buffer)) > 0) {
fos.write(buffer, 0, len);
}
fos.close();
byte[] bytes = fileConvert2ByteArray(file);
String value = Base64Encoder.encode(bytes);
String key = name.substring(0, name.lastIndexOf("."));
param.put(key, value);
// 删除临时文件
file.delete();
log.info("解压zip结束线程ID:<{}>结束运行,",Thread.currentThread().getId());
} catch (IOException e) {
log.error("[unZip()--解压异常2] Message={}",e.getMessage());
}
}
}
private static class ExcelTask implements Callable<List<Map<String, Object>>> {
private Workbook workbook;
private Integer startRow;
private Integer endRow;
private Semaphore semaphore;
private CountDownLatch countDownLatch;
private ReadXlsxPicturesUtil readXlsxPicturesUtil;
public ExcelTask(Semaphore semaphore,
Workbook workbook,
Integer startRow,
Integer endRow,
CountDownLatch countDownLatch,
ReadXlsxPicturesUtil readXlsxPicturesUtil){
this.workbook = workbook;
this.startRow = startRow;
this.endRow = endRow;
this.semaphore = semaphore;
this.countDownLatch = countDownLatch;
this.readXlsxPicturesUtil = readXlsxPicturesUtil;
}
@Override
public List<Map<String, Object>> call() throws Exception {
log.info("线程ID:<{}>开始运行,startRow:{},endRow:{}",Thread.currentThread().getId(),startRow,endRow);
semaphore.acquire();
log.info("消耗了一个信号量,剩余信号量为:{}",semaphore.availablePermits());
countDownLatch.countDown();
Sheet sheet = workbook.getSheetAt(0);
StringBuilder sb = new StringBuilder();
List<Map<String, Object>> list = new ArrayList<>();
int lastCell = sheet.getRow(1).getLastCellNum();//得到列数,默认数据从excel第2行开始
for(int i = startRow; i <= endRow; i++){
Row row = sheet.getRow(i);
if (row == null) {
continue;
}
Map<String, Object> param = new HashMap<>();
for (int j = 0; j < lastCell; j++) {
Cell cell = row.getCell(j);
String value = getCellValueText(cell);
if (value != null) {
value = value.trim();
}
if (j == 0) {
if (StringUtils.isEmpty(value)) {
}
if (!StringUtil.isEmpty(value) && value.length() > 30) {
}
sb.append(value).append("_").append(i).append(".").append(j).append("|");
param.put("你的姓名", value);
}
if (j == 1) {
if (StringUtils.isEmpty(value)) {
throw new BaseBackRuntimeException(ResultEnum.EXIST_REMARK);
}
if (!StringUtil.isEmpty(value) && value.length() > 30) {
}
sb.append(value).append("_").append(i).append(".").append(j).append("|");
param.put("你的备注", value);
}
if (j == 2) {
if (StringUtils.isEmpty(value)) {
}
param.put("你的类型", value);
}
}
if (!CollectionUtils.isEmpty(param)) {
list.add(param);
}
}
semaphore.release();//释放并将其返回给信号量。
if (你的对象.checkWord(sb.toString())) {
}
return list;
}
}
private static class UnRarFileTask implements Callable<Map<String, Object>> {
private IInArchive archive;
private int[] arr;
private Map<String, Object> param;
private File file;
private CountDownLatch countDownLatch;
private RandomAccessFile randomAccessFile;
public UnRarFileTask(IInArchive archive, int[] arr, Map<String, Object> param, File file,
CountDownLatch countDownLatch, RandomAccessFile randomAccessFile) {
this.archive = archive;
this.arr = arr;
this.param = param;
this.file = file;
this.countDownLatch = countDownLatch;
this.randomAccessFile = randomAccessFile;
}
public UnRarFileTask(RandomAccessFile randomAccessFile,IInArchive archive,Map<String, Object> param, File file,
CountDownLatch countDownLatch){
this.randomAccessFile = randomAccessFile;
this.archive = archive;
this.param = param;
this.file = file;
this.countDownLatch = countDownLatch;
}
@Override
public Map<String, Object> call() throws Exception {
countDownLatch.countDown();
//randomAccessFile = new RandomAccessFile(file, "rw");
//RandomAccessFileInStream randomAccessFileInStream = new RandomAccessFileInStream(randomAccessFile);
//archive = SevenZip.openInArchive(null,randomAccessFileInStream);
int[] arr = new int[archive.getNumberOfItems()];
//判断图片数量
if (arr.length >= 101) {
throw new BaseBackRuntimeException(ResultEnum.SIZE_EXIST_COUNT);
}
for (int i = 0; i < arr.length; i++) {
arr[i] = i;
}
IArchiveExtractCallback extractCallback = getExtractCallback(archive, file, null, param);
archive.extract(arr, false, extractCallback);
return param;
}
}
public static void main(String [] args) throws Exception {
List<Map<String, Object>> list = 你的Util.getXlsxList(files);
}
}
多线程解压缩包
于 2023-07-10 17:20:20 首次发布
![](https://img-home.csdnimg.cn/images/20240711042549.png)