https方式下载压缩文件,直接读取压缩文件内容 ,处理字节流等操作
最近有个需求,https方式下载文件,读取文件内容解析并落库,同时将文件上传到中台系统。花费了一点时间。因此整理了一下,以供有类似需求的同学参考
直接贴工具类吧~具体业务逻辑自己补充即可。
package com.canggogo.lv.util;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.fileupload.FileItem;
import org.apache.commons.fileupload.FileItemFactory;
import org.apache.commons.fileupload.disk.DiskFileItemFactory;
import org.springframework.http.MediaType;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Component;
import org.springframework.web.multipart.MultipartFile;
import org.springframework.web.multipart.commons.CommonsMultipartFile;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
/**
* @ClassName FileDownLoad
* @Author lyw
* @Description 通过url下载文件并解析处理
* @Date 2021/9/23 19:25
* @Version 1.0
**/
@Component
@Slf4j
@Async
public class FileDownLoad {
/**
* 将InputStream中的字节保存到ByteArrayOutputStream中。
*/
private ByteArrayOutputStream byteArrayOutputStream = null;
private String midFileId;
private String transDate;
private String merchantCode;
private String downloadUrl;
private String fileName;
private String nowDate;
private String merId;
private Long billId;
private String downloadType;
@Async
public void downLoad(String url) {
try {
decompress(url, fileName);
} catch (Exception e) {
e.printStackTrace();
log.error( e.getMessage());
}
}
/**
* 解析文件
*
* @param url
* @param fileName
* @return
* @throws Exception
*/
public void decompress(String url, String fileName) {
try {
long start = System.currentTimeMillis();
HttpURLConnection conn = (HttpURLConnection) new URL(url).openConnection();
conn.setReadTimeout(30000);
conn.setConnectTimeout(30000);
//设置应用程序要从网络连接读取数据
conn.setDoInput(true);
conn.setRequestMethod("GET");
if (conn.getResponseCode() == HttpURLConnection.HTTP_OK) {
InputStream is = conn.getInputStream();
//缓存io流,可以重复读取,读取三个Stream流分别用于上传,本地保存,解析
InputStreamCacher(is);
InputStream uploadStream = getInputStream();
InputStream decompressStream = getInputStream();
InputStream downLoadInputStream = getInputStream();
//下载文件,保存到服务器。
File file = downFileToLocal(downLoadInputStream);
//将流转为multpartFile,可用于上传
MultipartFile multipartFile = getMultipartFile(uploadStream, fileName);
//直接读取压缩文件流中的内容,解析数据落库
readFile(decompressStream);
long end = System.currentTimeMillis();
log.info("文件处理完成,消耗时间:{}毫秒", end - start);
}
} catch (IOException e) {
throw new RuntimeException("文件下载失败", e);
}
}
private File downFileToLocal(InputStream downLoadInputStream) {
File file = null;
FileOutputStream fileOut;
String path = this.getClass().getClassLoader().getResource("").getPath();
path =path+"012345/";
try {
ZipInputStream zipInputStream = new ZipInputStream(downLoadInputStream);
ByteArrayOutputStream bos = new ByteArrayOutputStream();
ZipEntry nextEntry;
while ((nextEntry = zipInputStream.getNextEntry()) != null) {
String name = nextEntry.getName();
file = new File(path);
//清除历史文件,重新创建
if (file.exists()) {
deleteFile(file);
}
//目录不存在 则创建
if (!file.exists()) {
boolean mkdir = file.mkdirs();
if (!mkdir) {
throw new RuntimeException("创建目标文件所在目录失败!");
}
}
file=new File(path+name);
file.createNewFile();
fileOut = new FileOutputStream(path + name);
merchantCode = name.split("_")[0];
int size = 0;
byte[] buffer = new byte[1024];
while (true) {
size = zipInputStream.read(buffer, 0, buffer.length);
if (size <= 0) {
break;
}
bos.write(buffer, 0, size);
}
fileOut.write(bos.toByteArray());
fileOut.close();
bos.flush();
bos.close();
}
zipInputStream.close();
} catch (Exception e) {
e.printStackTrace();
}
return file;
}
/**
* @return
* @Description 读取压缩文件,解析内容
* @Date 2021/9/24 15:06
* @Param [inputStream]
**/
public void readFile(InputStream inputStream) throws IOException {
List<String[]> lines = new ArrayList<>();
ByteArrayOutputStream bos = new ByteArrayOutputStream();
ZipInputStream zipInputStream = new ZipInputStream(inputStream);
String rtn = null;
ZipEntry nextEntry;
try {
while ((nextEntry = zipInputStream.getNextEntry()) != null) {
String name = nextEntry.getName();
log.info("文件名:"+name);
int size = 0;
byte[] buffer = new byte[1024];
while (true) {
size = zipInputStream.read(buffer, 0, buffer.length);
if (size <= 0) {
break;
}
bos.write(buffer, 0, size);
}
bos.flush();
bos.close();
}
zipInputStream.close();
} catch (Exception e) {
e.printStackTrace();
} finally {
try {
byte[] be = bos.toByteArray();
rtn = new String(be, "utf-8");
bos.flush();
bos.close();
zipInputStream.close();
} catch (Exception e) {
e.printStackTrace();
}
}
if (rtn != null) {
rtn = rtn.replaceAll("\\r", "");
String[] split = rtn.split("\n");
List<String> list = Arrays.asList(split);
for (int i = 0; i < list.size(); i++) {
//用换行符切割字符串获取每行数据
String[] split1 = list.get(i).split("\\|");
lines.add(split1);
}
//批量落地对账数据
// batchInsertRecords(lines);
}
}
public void InputStreamCacher(InputStream inputStream) {
if (inputStream==null){
return;
}
byteArrayOutputStream = new ByteArrayOutputStream();
byte[] buffer = new byte[1024];
int len;
try {
while ((len = inputStream.read(buffer)) > -1) {
byteArrayOutputStream.write(buffer, 0, len);
}
byteArrayOutputStream.flush();
} catch (IOException e) {
e.printStackTrace();
}
}
public InputStream getInputStream() {
if (byteArrayOutputStream==null) {
return null;
}
return new ByteArrayInputStream(byteArrayOutputStream.toByteArray());
}
/**
* 获取封装得MultipartFile
*
* @param inputStream inputStream
* @param fileName fileName
* @return MultipartFile
*/
public MultipartFile getMultipartFile(InputStream inputStream, String fileName) {
FileItem fileItem = createFileItem(inputStream, fileName);
//CommonsMultipartFile是feign对multipartFile的封装,但是要FileItem类对象
return new CommonsMultipartFile(fileItem);
}
/**
* FileItem类对象创建
*
* @param inputStream inputStream
* @param fileName fileName
* @return FileItem
*/
public FileItem createFileItem(InputStream inputStream, String fileName) {
FileItemFactory factory = new DiskFileItemFactory(16, null);
String textFieldName = "chinaPay";
FileItem item = factory.createItem(textFieldName, MediaType.MULTIPART_FORM_DATA_VALUE, false, fileName);
int bytesRead = 0;
byte[] buffer = new byte[10 * 1024 * 1024];
OutputStream os = null;
//使用输出流输出输入流的字节
try {
os = item.getOutputStream();
while ((bytesRead = inputStream.read(buffer, 0, 8192)) != -1) {
os.write(buffer, 0, bytesRead);
}
inputStream.close();
} catch (IOException e) {
log.error("Stream copy exception", e);
throw new IllegalArgumentException("文件上传失败");
} finally {
if (os != null) {
try {
os.close();
} catch (IOException e) {
log.error("Stream close exception", e);
}
}
if (inputStream != null) {
try {
inputStream.close();
} catch (IOException e) {
log.error("Stream close exception", e);
}
}
}
return item;
}
//删除某个文件夹下面的所有文件
public static void deleteFile(File file) {
//判断文件不为null或文件目录存在
if (file == null || !file.exists()) {
System.out.println("文件删除失败,请检查文件路径是否正确");
return;
}
//取得这个目录下的所有子文件对象
File[] files = file.listFiles();
//遍历该目录下的文件对象
for (File f : files) {
//打印文件名
String name = file.getName();
// System.out.println(name);
//判断子目录是否存在子目录,如果是文件则删除
if (f.isDirectory()) {
deleteFile(f);
} else {
f.delete();
}
}
//删除空文件夹 for循环已经把上一层节点的目录清空。
file.delete();
}
}