Windows环境执行成功,Linux服务器上执行不成功,可能的原因是: 服务器上是k8s部署,controller和service部署在不同容器中,定义的文件路径需要挂载到同一个地方(PVC) @ApiOperation(value = "上传rar文件") @PostMapping("uploadRarFile") public ResponseVO<String> uploadRarFile(@RequestParam("file") MultipartFile file, HttpServletRequest request, Integer catalogId) { File convertedFile = new File(file.getOriginalFilename()); try { String originalFilename = file.getOriginalFilename(); //判断上传文件必须是zip或者是rar否则不允许上传 String[] filename = originalFilename.split("\\."); if (!filename[1].equals("zip") && !filename[1].equals("rar")) { throw new ValidateException("文件格式校验失败!"); } convertedFile.createNewFile(); InputStream inputStream = file.getInputStream(); FileOutputStream outputStream = new FileOutputStream(convertedFile); FileCopyUtils.copy(inputStream, outputStream); inputStream.close(); outputStream.close(); return throwResponse(uploadRarFile(convertedFile, catalogId)); } catch (Exception e) { log.error("上传rar文件失败:" , e); return throwResponse(9999, "上传文件失败"+e.getMessage()); } } @ApiOperation(value = "下载SQL rar文件") @GetMapping("downloadRarFile") public void downloadRarFile(@RequestParam(name = "tableNames", required = false, defaultValue = "") List<String> tableNames, @RequestParam(name = "type") Integer type, HttpServletResponse response) { try { iCatalogService.downloadRarFile(tableNames, type); OutputStream toClient = null; File file = new File(fileName); try { // 以流的形式下载文件。 BufferedInputStream fis = new BufferedInputStream(new FileInputStream(file.getPath())); byte[] buffer = new byte[fis.available()]; fis.read(buffer); fis.close(); // 清空response response.reset(); toClient = new BufferedOutputStream(response.getOutputStream()); response.setCharacterEncoding("UTF-8"); response.setContentType("application/octet-stream"); response.setHeader("Access-Control-Allow-Origin", "*"); response.setHeader("Content-Disposition", "attachment;filename*=utf-8''" + file.getName()); toClient.write(buffer); toClient.flush(); } catch (Exception e) { log.error("下载zip压缩包过程发生异常:", e); } finally { if (toClient != null) { try { toClient.close(); } catch (IOException e) { log.error("zip包下载关流失败:", e); } } //删除该临时zip包(此zip包任何时候都不需要保留,因为源文件随时可以再次进行压缩生成zip包) file.delete(); //删除生成的SQL文件 deleteDirectory(Paths.get(sqlPathUnrar)); } } catch (Exception e) { log.error("下载rar文件失败:" , e); } } public ServiceVO<String> uploadRarFile(File file, Integer catalogId) { MysqlExecuteUtil mysqlExecuteUtil = new MysqlExecuteUtil(hostPDL, Integer.valueOf(portPDL), databasePDL, usernamePDL, passwordPDL); AtomicReference<String> fileName = new AtomicReference<>(""); List<String> successFiles = new ArrayList<>(); List<String> failFiles = new ArrayList<>(); String errorMsg = ""; Path directoryPath = Paths.get(sqlPathRar); try { List<String> filePath = FileUtil.unRar(file, sqlPathRar); if (CollectionUtils.isEmpty(filePath)) return ServiceUtils.throwFailure(9999, "文件内容为空"); Set<Long> set = new HashSet<>(); for (int i=0; i<filePath.size(); i++) { log.info("file path {}", filePath.get(i)); long count = filePath.get(i).chars().filter(ch -> ch == '/').count(); if (CollectionUtils.isEmpty(set)) { set.add(count); } else { if (set.add(count)) { return ServiceUtils.throwFailure(9998, "文件内容存在多个目录"); } } } filePath.stream().distinct().forEach(f-> { if (f.indexOf("rar") ==-1) { File file1 = new File(f); fileName.set(file1.getName()); List<String> line = new ArrayList<>(); try (BufferedReader bufferedReader = new BufferedReader(new FileReader(f))) { line = bufferedReader.lines().collect(Collectors.toList()); } catch (IOException e) { e.printStackTrace(); } String stringFromList = String.join(" ", line); String[] sql = stringFromList.split(";"); Arrays.stream(sql).forEach(s->{ mysqlExecuteUtil.execute(s); SQLCreateTableStatement sqlCreateTableStatement = (SQLCreateTableStatement) SQLUtils.parseStatements(s, JdbcConstants.MYSQL).get(0); String tableName = sqlCreateTableStatement.getTableName(); tableName = tableName.replace("`", ""); CatalogSourcePO catalogSourcePO = new CatalogSourcePO(); catalogSourcePO.setTableName("pdl."+tableName); catalogSourcePO.setParentId(catalogId); iCatalogService.createCatalogSource(catalogSourcePO); }); successFiles.add(fileName.get()); } }); } catch (Exception e) { e.printStackTrace(); failFiles.add(fileName.get()); errorMsg = e.getMessage(); } finally { try { deleteDirectory(directoryPath); } catch (IOException e) { throw new RuntimeException(e); } } String result = String.format("successList = %s \nfailList = %s \nerrorMsg = %s", successFiles, failFiles, errorMsg); if (StringUtils.isNotEmpty(errorMsg)) return ServiceUtils.throwFailure(9999, result); else return ServiceUtils.throwResponse(result); } 删除多级目录及文件 private void deleteDirectory(Path path) throws IOException { if (Files.isDirectory(path, LinkOption.NOFOLLOW_LINKS)) { try (DirectoryStream<Path> entries = Files.newDirectoryStream(path)) { for (Path entry : entries) { deleteDirectory(entry); } } } Files.delete(path); }
Service层处理逻辑:
1.导出表结构
2.导出数据(大数据分批导出)
3.压缩导出的SQL文件
@Override public void downloadRarFile(List<String> tableNames, Integer type) { MysqlExecuteUtil mysqlExecuteUtil = new MysqlExecuteUtil(hostPDL, Integer.valueOf(portPDL), databasePDL, usernamePDL, passwordPDL); BufferedWriter writer = null; CatalogSourceListPO catalogSourceListPO = new CatalogSourceListPO(); catalogSourceListPO.setGroup(1); List<CatalogSourcePO> catalogSourcePOS = catalogSourceDao.queryAllCatalogSource(catalogSourceListPO); if (!CollectionUtils.isEmpty(tableNames)) catalogSourcePOS = catalogSourcePOS.stream().filter(c->tableNames.contains(c.getTableName())).collect(Collectors.toList()); for (CatalogSourcePO c:catalogSourcePOS) { try { String tableName = c.getTableName(); tableName = tableName.substring(tableName.indexOf(".") + 1); ArrayList<String> pathNames = new ArrayList<>(); if (c.getParentId() != null) { CatalogPO catalogPO = iCatalogDao.selectDataFieldById(c.getParentId()); pathNames.add(catalogPO.getName()); while (catalogPO.getParentId() != null) { catalogPO = iCatalogDao.selectDataFieldById(catalogPO.getParentId()); pathNames.add(catalogPO.getName()); } } Collections.reverse(pathNames); String pathName = String.join("/", pathNames); String fileName = ""; if (StringUtils.isNotEmpty(pathName)) { fileName = sqlPathUnrar + pathName + "/" + tableName + ".sql"; } else { fileName = sqlPathUnrar + tableName + ".sql"; } File file = new File(fileName); log.info("file name {}", fileName); if (!file.isFile() && !file.exists()) { //判断文件是否存在 log.info("file name1 {}", fileName.lastIndexOf("/")); if (fileName.lastIndexOf("/") !=-1) { //windows和Linux此处'/'判断不同 String path = fileName.substring(0, fileName.lastIndexOf("/")); new File(path).mkdirs(); String name = fileName.substring(fileName.lastIndexOf("/") + 1); new File(name).createNewFile(); } } writer = new BufferedWriter(new FileWriter(file)); // 导出建表语句 String createTableSQL = mysqlExecuteUtil.getCreateTableSQL("`"+tableName+"`"); writer.write(createTableSQL + ";"); writer.newLine(); if (type == 2) { // 导出插入数据语句 需分批导出 ResultSet resultSet = mysqlExecuteUtil.executeSql("select count(*) from " + "`"+tableName+"`"); resultSet.next(); int count = resultSet.getInt(1); //数据量分割值 int num = 1000; //循环次数/分割次数 int cycles = count / num; //余数 int remainder = count % num; boolean flag = true; //分批查询次数 for (int i = 0; i < cycles; i++) { //数据超过1w不用导出 if (i * num > 10000) { flag = false; break; } String insertDataSQL = mysqlExecuteUtil.getInsertDataSQL(tableName, i * num, num); writer.write(insertDataSQL); writer.newLine(); } if (remainder > 0 && flag) { String insertDataSQL = mysqlExecuteUtil.getInsertDataSQL(tableName, num * cycles, (num * cycles) + remainder); writer.write(insertDataSQL); writer.newLine(); } } } catch (IOException e) { throw new RuntimeException(e); } catch (SQLException e) { throw new RuntimeException(e); } finally { // 关闭资源 try { if (writer != null) writer.close(); } catch (IOException e) { e.printStackTrace(); } } } try { FileUtil.compress(new File(sqlPathUnrar), new File(fileName)); } catch (IOException e) { throw new RuntimeException(e); } catch (ArchiveException e) { throw new RuntimeException(e); } }
FileUtil:解压rar和压缩文件
package com.zhanwan.centre.rest.utils; import lombok.extern.slf4j.Slf4j; import net.sf.sevenzipjbinding.IInArchive; import net.sf.sevenzipjbinding.SevenZip; import net.sf.sevenzipjbinding.impl.RandomAccessFileInStream; import org.apache.commons.compress.archivers.ArchiveEntry; import org.apache.commons.compress.archivers.ArchiveException; import org.apache.commons.compress.archivers.ArchiveOutputStream; import org.apache.commons.compress.archivers.ArchiveStreamFactory; import java.io.*; import java.util.ArrayList; import java.util.List; @Slf4j public class FileUtil { private static final int BUFFER_SIZE = 2048; public static List<String> unRar(File rarFile, String dstDirectoryPath){ IInArchive archive; RandomAccessFile randomAccessFile; try { randomAccessFile = new RandomAccessFile(rarFile,"r"); archive = SevenZip.openInArchive(null,new RandomAccessFileInStream(randomAccessFile)); int[] in = new int[archive.getNumberOfItems()]; for (int i = 0; i < in.length; i++) { in[i] = i; } archive.extract(in,false,new ExtractCallback(archive,dstDirectoryPath)); archive.close(); randomAccessFile.close(); log.info("解压目标文件夹:{}", dstDirectoryPath); List<String> allFileList = getAllFile(dstDirectoryPath,false); log.info("allFileList:{}", allFileList); ArrayList<String> resultFileList = new ArrayList<>(); String startString; String endString; String finallyString; for (String s : allFileList) { if (s.startsWith("/")||s.startsWith("\\")) { startString = s.substring(0,s.lastIndexOf("\\")); endString = s.substring(s.lastIndexOf("\\")+1); finallyString = startString+"\\"+endString; }else { // windows去掉盘符 s.substring(2); startString = s.substring(0,s.lastIndexOf("\\")); endString = s.substring(s.lastIndexOf("\\")+1); finallyString = startString+"/"+endString; } // 解决linux路径出现//导致文件路径错误 finallyString = finallyString.replaceAll("//","/"); resultFileList.add(finallyString); } log.info("rar文件解压文件路径为:{}", resultFileList); return resultFileList; } catch (Exception e) { e.printStackTrace(); } return null; } public static List<String> getAllFile(String directoryPath,boolean isAddDirectory){ List<String> list = new ArrayList<>(); File baseFile = new File(directoryPath); if (baseFile.isFile()||!baseFile.exists()) { return list; } File[] files = baseFile.listFiles(); for (File file : files) { if (file.isDirectory()) { if (isAddDirectory) { list.add(file.getAbsolutePath()); } list.addAll(getAllFile(file.getAbsolutePath(),isAddDirectory)); }else { list.add(file.getAbsolutePath()); } } return list; } public static void compress(File sourceFile, File targetFile) throws IOException, ArchiveException { // 如果目标文件不存在,创建一个 if (!targetFile.exists()) { targetFile.createNewFile(); } OutputStream outputStream = new FileOutputStream(targetFile); ArchiveOutputStream archiveOutputStream = null; try { // 创建 RAR 归档输出流 archiveOutputStream = new ArchiveStreamFactory().createArchiveOutputStream("zip", outputStream); // 打包文件 compressRecursive(sourceFile, "", archiveOutputStream); } finally { // 关闭归档输出流 if (archiveOutputStream != null) { archiveOutputStream.finish(); } // 关闭输出流 if (outputStream != null) { outputStream.close(); } } } private static void compressRecursive(File file, String path, ArchiveOutputStream archiveOutputStream) throws IOException { String fileName = file.getName(); String fileFullName = path + fileName; // 如果是目录,递归打包 if (file.isDirectory()) { ArchiveEntry entry = archiveOutputStream.createArchiveEntry(file, fileFullName + "/"); archiveOutputStream.putArchiveEntry(entry); archiveOutputStream.closeArchiveEntry(); File[] files = file.listFiles(); for (File subFile : files) { compressRecursive(subFile, fileFullName + "/", archiveOutputStream); } } else { // 如果是文件,直接打包 InputStream inputStream = new FileInputStream(file); ArchiveEntry entry = archiveOutputStream.createArchiveEntry(file, fileFullName); archiveOutputStream.putArchiveEntry(entry); byte[] buffer = new byte[BUFFER_SIZE]; int length; while ((length = inputStream.read(buffer)) != -1) { archiveOutputStream.write(buffer, 0, length); } archiveOutputStream.closeArchiveEntry(); inputStream.close(); } } }