FTP本地文件上传、远程文件下载
mq消费端
1,mq接收端，监听sxqxgxw_sync_radar_img的消息
2,创建FtpHelper a实例,连接远程资源服务器a
3,创建FtpHelper b实例,连接远程资源服务器b
4,获取远程资源服务器a,目标路径下文件,可以使用ftpFileFilter过滤获取到的文件列表
5,如果本地不存在，远程文件
6,使用ftp下载远程文件
7,判断下载好的文件大小是否为0,如果是0删除
8,如果本地下载成功
9,读取远程服务器b路径下文件,是否包含刚下载好的本地文件
10,上传本地文件到远程服务器b
11,查看远程服务器b目录下刚上传的文件大小是否为0,如果为0删除
12,将上传或者下载的文件当成对象存入集合并且去重
13,将下载文件自定义对象存入虚谷数据库
14,将上传图片的自定义对象存入pgsql中
package com.sxqx.listener.impl;
import com.alibaba.fastjson.JSONObject;
import com.fasterxml.jackson.databind.JsonNode;
import com.sxqx.entity.MQMessage;
import com.sxqx.mapper.remote.pgsql.radar.IRadarImagePgSqlMapper;
import com.sxqx.mapper.remote.xugu1.radar.IRadarImageMapper;
import com.sxqx.pojo.RadarImageItem;
import com.sxqx.listener.IMessageReceiver;
import com.sxqx.utils.common.ListUtils;
import com.sxqx.utils.dataConverter.JsonConverter;
import com.sxqx.utils.exception.MessageUtils;
import com.sxqx.utils.filter.ftp.RadarImageFilter;
import com.sxqx.utils.ftp.FtpHelper;
import com.sxqx.utils.mq.MQMessageSender;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.net.ftp.FTPFileFilter;
import org.springframework.amqp.rabbit.annotation.Queue;
import org.springframework.amqp.rabbit.annotation.RabbitHandler;
import org.springframework.amqp.rabbit.annotation.RabbitListener;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.io.File;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.stream.Collectors;
@Component
public class SyncRadarImageMessageReceiver implements IMessageReceiver {
private final IRadarImageMapper radarImageMapper;
private final IRadarImagePgSqlMapper radarImagePgSqlMapper;
private final MQMessageSender mqMessageSender;
@Autowired
public SyncRadarImageMessageReceiver(MQMessageSender mqMessageSender,
IRadarImagePgSqlMapper radarImagePgSqlMapper,
IRadarImageMapper radarImageMapper) {
this.mqMessageSender = mqMessageSender;
this.radarImagePgSqlMapper = radarImagePgSqlMapper;
this.radarImageMapper = radarImageMapper;
}
Log log = LogFactory.getLog(SyncRadarImageMessageReceiver.class);
FtpHelper ftpHelperIns1;
FtpHelper ftpHelperIns2;
@RabbitListener(queuesToDeclare = {
@Queue(name = "sxqxgxw_sync_radar_img")
})
@RabbitHandler
@Override
public void onMessageReceived(String mqMessageString) {
JsonNode jsonNode = JsonConverter.jsonString2JsonNode(mqMessageString);
JsonNode msg = jsonNode.findValue("msg");
JsonNode JsonNodeParams = msg.findValue("params");
Map<String, Object> params = JsonConverter.jsonNode2HashMap(JsonNodeParams);
if (params.size() > 0) {
String server = params.get("server").toString();
int port = Integer.parseInt(params.get("port").toString());
String username = params.get("username").toString();
String passwd = params.get("passwd").toString();
String[] cities = new String[]{"xian", "yanan", "yulin", "baoji", "shangluo", "ankang", "hanzhong", "baoji"};
try {
//远程服务器1 ftp同步数据
ftpHelperIns1 = FtpHelper.getInstance(server, port, username, passwd, "instance1");
boolean connIns1 = ftpHelperIns1.connect();
if (!connIns1) {
return;
}
ftpHelperIns2 = FtpHelper.getInstance("远程服务器2", 21, "ftp_user", "密码", "instance2");
boolean connIns2 = ftpHelperIns2.connect();
if (!connIns2) {
return;
}
for (String city : cities) {
String localPath = params.get("localPath").toString() + city + "/";
String remoteFtpPath = params.get("remoteFtpPath").toString() + city + "/";
//过滤1小时内数据
FTPFileFilter ftpFileFilter = new RadarImageFilter();
List<String> fileList = ftpHelperIns1.getFileList(remoteFtpPath, ftpFileFilter);
List<RadarImageItem> radarImageItemList = new ArrayList<>();
List<RadarImageItem> radarImageItemPgSqlList = new ArrayList<>();
for (String file : fileList) {
//过滤hail.jpg、heavyrain.jpg、thunderstorm.jpg文件，因为比如Z9911_20230806000103Z_UAM_00_hail.jpg会导致126行转换异常，并且不需要这些图片
if (!new File(localPath + file).exists()
&& !file.endsWith("hail.jpg")
&& !file.endsWith("heavyrain.jpg")
&& !file.endsWith("thunderstorm.jpg")
&& !file.endsWith("tornado.jpg")) {
boolean res = ftpHelperIns1.downloadFile(localPath, remoteFtpPath, file);
//判断下载好的文件大小是否为0
if (new File(localPath + file).exists() && new File(localPath + file).length() == 0) {
new File(localPath + file).delete();
log.info("remove empty file under : " + localPath + file);
res = false;
}
if (res) {
boolean res1 = false;
if (file.contains("CR_00_037") || file.contains("CR_00_038")) {
if (city.equals("xian") || city.equals("baoji")) {
//读取指定目录下的文件名
List<String> fileList1 = ftpHelperIns2.getFileList("/static/resource/radar/" + city, ftpFileFilter);
//远程目录不包含file时
if (fileList1 != null && !fileList1.contains(file)) {
//上传图片到远程服务器2
res1 = ftpHelperIns2.uploadFile(localPath, "/static/resource/radar/" + city, file);
if (res1) {
boolean removed = ftpHelperIns2.removeEmptyFile("/static/resource/radar/" + city, file, ftpFileFilter);
if (removed) {
log.info("remove empty file under ftp server 远程服务器2 /data/static/resource/radar/" + city + "/" + file);
res1 = false;
}
}
}
}
}
//Z9290_20230714043718Z_PPI_05_019.jpg
String[] s = file.split("_");
if (s.length == 5) {
int degree = Integer.parseInt(s[3]);
String product = s[2];
Integer number = Integer.parseInt(s[4].replaceAll(".jpg", ""));
String station = s[0].replaceAll("Z9", "");
String dt = s[1].substring(0, 14);
SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmmss");
Date date = sdf.parse(dt);
Calendar bjc = Calendar.getInstance();
bjc.setTime(date);
bjc.add(Calendar.HOUR_OF_DAY, 8);
date = bjc.getTime();
RadarImageItem radarImageItem = new RadarImageItem();
radarImageItem.setTime(date);
radarImageItem.setDegree(degree);
radarImageItem.setNumber(number);
radarImageItem.setStation(station);
radarImageItem.setPath("http://本地服务器/mediaResource/RadarPic/" + city + "/" + file);
radarImageItem.setFilename(file);
radarImageItem.setProduct(product);
radarImageItemList.add(radarImageItem);
if (res1) {
RadarImageItem radarImagePgsqlItem = new RadarImageItem();
radarImagePgsqlItem.setTime(date);
radarImagePgsqlItem.setDegree(degree);
radarImagePgsqlItem.setNumber(number);
radarImagePgsqlItem.setStation(station);
radarImagePgsqlItem.setPath("http://远程服务器2:5000/resource/radar/" + city + "/" + file);
radarImagePgsqlItem.setFilename(file);
radarImagePgsqlItem.setProduct(product);
//radarImagePgsqlItem.setExtent("");
radarImageItemPgSqlList.add(radarImagePgsqlItem);
}
// 清除数据
MQMessage mqMessage = new MQMessage();
JSONObject jsonObject = new JSONObject();
jsonObject.put("filePath", "/media/resource/RadarPic/" + city + "/" + file);
mqMessage.setMsg(jsonObject);
mqMessageSender.send("queue.file_delay_destroy", mqMessage);
}
}
}
}
List<RadarImageItem> distinctClass = radarImageItemList.stream()
.collect(Collectors.collectingAndThen(Collectors.toCollection(
() -> new TreeSet<>(Comparator.comparing(
o -> o.getStation() + ";" + o.getDegree() + ";" + o.getNumber() + ";" + o.getTime()))), ArrayList::new));
List<RadarImageItem> distinctClassPgSql = radarImageItemPgSqlList.stream()
.collect(Collectors.collectingAndThen(Collectors.toCollection(
() -> new TreeSet<>(Comparator.comparing(
o -> o.getStation() + ";" + o.getDegree() + ";" + o.getNumber() + ";" + o.getTime()))), ArrayList::new));
if (distinctClass.size() > 0) {
for (List<RadarImageItem> radarImageItems : ListUtils.splitList(distinctClass, 50)) {
int rows = radarImageMapper.insertIntoRadarImage(radarImageItems);
Thread.sleep(50);
}
Thread.sleep(1000);
}
//pgsql插入数据库
if (distinctClassPgSql.size() > 0) {
for (List<RadarImageItem> radarImageItems : ListUtils.splitList(distinctClassPgSql, 10)) {
int rows = radarImagePgSqlMapper.insertIntoRadarImage(radarImageItems);
Thread.sleep(50);
}
Thread.sleep(1000);
}
}
} catch (Exception ex) {
//ex.printStackTrace();
log.error(MessageUtils.getErrMsg(ex));
} finally {
//关闭资源
if (ftpHelperIns1.connect()) {
ftpHelperIns1.close();
}
if (ftpHelperIns2.connect()) {
ftpHelperIns2.close();
}
}
}
}
}
ftp工具类包含以上用到的建立连接、查找文件、下载文件、删除空文件、上传文件
package com.sxqx.utils.ftp;
import com.sxqx.utils.exception.MessageUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.net.ftp.*;
import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
public class FtpHelper {
private final FTPClient ftp;
private static String _ftpServer;
private static int _ftpPort;
private static String _ftpUsername;
private static String _ftpPassword;
private FTPClientStatus status;
private static FtpHelper instance;
Log log = LogFactory.getLog(FtpHelper.class);
public static FtpHelper getInstance(String ftpServer, int ftpPort, String ftpUsername, String ftpPassword) {
if (null == instance) {
_ftpServer = ftpServer;
_ftpPort = ftpPort;
_ftpUsername = ftpUsername;
_ftpPassword = ftpPassword;
instance = new FtpHelper();
}
return instance;
}
public FtpHelper() {
ftp = new FTPClient();
status = FTPClientStatus.DISCONNECTED;
}
/**
* ftp连接服务器
*/
public boolean connect() {
try {
ftp.setDataTimeout(10000);
ftp.setConnectTimeout(10000);
ftp.connect(_ftpServer, _ftpPort);
ftp.login(_ftpUsername, _ftpPassword);
if (FTPReply.isPositiveCompletion(ftp.getReplyCode())) {
status = FTPClientStatus.CONNECTED;
return true;
} else {
status = FTPClientStatus.CONNECT_EXCEPTION;
return false;
}
} catch (Exception e) {
log.error(MessageUtils.getErrMsg(e));
status = FTPClientStatus.CONNECT_EXCEPTION;
return false;
}
}
/**
* 读取指定目录下的文件名
*
* @param path 路径名称
*/
public List<String> getFileList(String path, FTPFileFilter filter) {
List<String> fileLists = new ArrayList<>();
FTPFile[] ftpFiles = null;
if (status == FTPClientStatus.CONNECTED) {
try {
String remotePath = ftp.printWorkingDirectory();
if (!Objects.equals(path, remotePath)) {
ftp.changeWorkingDirectory(path);
}
ftp.enterLocalPassiveMode();
if (filter != null) {
ftpFiles = ftp.listFiles(path, filter);
} else {
ftpFiles = ftp.listFiles(path);
}
} catch (IOException e) {
//e.printStackTrace();
log.error(MessageUtils.getErrMsg(e));
}
for (int i = 0; ftpFiles != null && i < ftpFiles.length; i++) {
FTPFile file = ftpFiles[i];
if (file.isFile()) {
fileLists.add(file.getName());
} else if (file.isDirectory()) {
List<String> childLists = getFileList(path + file.getName() + "/", filter);
fileLists.addAll(childLists);
}
}
}
return fileLists;
}
/**
* 下载文件到指定目录
*
* @param localPath 本地路径
* @param ftpPath FTP路径
* @param fileName 文件名
*/
public boolean downloadFile(String localPath, String ftpPath, String fileName) {
if (status == FTPClientStatus.CONNECTED) {
OutputStream outputStream = null;
try {
ftp.setFileType(FTP.BINARY_FILE_TYPE);
ftp.enterLocalPassiveMode();
String remotePath = ftp.printWorkingDirectory();
if (!Objects.equals(ftpPath, remotePath)) {
ftp.changeWorkingDirectory(ftpPath);
}
File localPathDir = new File(localPath);
if (!localPathDir.exists()) {
boolean res = localPathDir.mkdirs();
}
File localFile = new File(localPath + fileName);
outputStream = Files.newOutputStream(localFile.toPath());
ftp.retrieveFile(fileName, outputStream);
outputStream.close();
return true;
} catch (IOException e) {
log.error(MessageUtils.getErrMsg(e));
return false;
}
}
return false;
}
/**
* 关闭连接
*/
public void close(){
try {
if(ftp.isConnected()){
ftp.logout();
ftp.disconnect();
}
}catch (IOException e){
e.printStackTrace();
}
}
/**
* 上传文件到指定目录
*
* @return
*/
public boolean uploadFile(String localPath, String remoteFilePath, String file) {
try {
String pathFile = localPath + file;
ftp.setFileType(ftp.BINARY_FILE_TYPE);
ftp.enterLocalPassiveMode();
String remotePath = ftp.printWorkingDirectory();
if (!Objects.equals(remoteFilePath, remotePath)) {
ftp.changeWorkingDirectory(remoteFilePath);
}
pathFile = new String(pathFile.getBytes("GBK"), "iso-8859-1");
//打开本地文件输入流
FileInputStream inputStream = new FileInputStream(pathFile);
// 上传文件到远程服务器
boolean uploaded = ftp.storeFile(new File(pathFile).getName(), inputStream);
inputStream.close();
if (uploaded) {
return true;
} else {
log.info("失败，服务器返回:" + ftp.getReplyString());//获取上传失败的原因
return false;
}
} catch (IOException e) {
return false;
}
}
/**
* 清除大小为0的文件
* @param remotePath
* @param fileName
* @param filter
* @return
*/
public boolean removeEmptyFile(String remotePath,String fileName, FTPFileFilter filter) {
boolean deleteFile = false;
String remotePathFileName;
if(!remotePath.endsWith("/")){
remotePathFileName = remotePath + "/" + fileName;
}else{
remotePathFileName = remotePath + fileName;
}
FTPFile[] ftpFiles;
if (status == FTPClientStatus.CONNECTED) {
try {
String remoteWorkingPath = ftp.printWorkingDirectory();
if (!Objects.equals(remoteWorkingPath, remotePath)) {
ftp.changeWorkingDirectory(remotePath);
}
ftp.enterLocalPassiveMode();
if (filter != null) {
ftpFiles = ftp.listFiles(remotePath, filter);
} else {
ftpFiles = ftp.listFiles(remotePath);
}
for (int i = 0; ftpFiles != null && i < ftpFiles.length; i++) {
FTPFile file = ftpFiles[i];
if (file.isFile()) {
if(Objects.equals(file.getName(),fileName)){
if(file.getSize()==0){
deleteFile = ftp.deleteFile(remotePathFileName);
}
}
}
}
} catch (IOException e) {
e.printStackTrace();
}
}
return deleteFile;
}
}
用天擎返回值图片url打开流，使用ftp上传图片到远程服务器
mq消费端
package com.sxqx.listener.impl;
import com.fasterxml.jackson.databind.JsonNode;
import com.sxqx.listener.IMessageReceiver;
import com.sxqx.mapper.remote.pgsql.radar.IRadarImagePgSqlMapper;
import com.sxqx.mapper.remote.xugu1.radar.IRadarImageMapper;
import com.sxqx.pojo.RadarImageItem;
import com.sxqx.pojo.cmadaas.respdata.RadaBexaCrNul1kmPngRetData;
import com.sxqx.service.weatherlive.radar.IRadarImage;
import com.sxqx.utils.common.ListUtils;
import com.sxqx.utils.dataConverter.JsonConverter;
import com.sxqx.utils.exception.MessageUtils;
import com.sxqx.utils.ftp.FtpHelper;
import com.sxqx.utils.mq.MQMessageSender;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.amqp.rabbit.annotation.Queue;
import org.springframework.amqp.rabbit.annotation.RabbitHandler;
import org.springframework.amqp.rabbit.annotation.RabbitListener;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.text.SimpleDateFormat;
import java.util.*;
@Component
public class SyncCmadaasRadaImgReceiver implements IMessageReceiver {
private final IRadarImageMapper radarImageMapper;
private final IRadarImagePgSqlMapper radarImagePgSqlMapper;
private final MQMessageSender mqMessageSender;
private IRadarImage radarImage;
@Autowired
public SyncCmadaasRadaImgReceiver(MQMessageSender mqMessageSender,
IRadarImagePgSqlMapper radarImagePgSqlMapper,
IRadarImageMapper radarImageMapper,
IRadarImage radarImage) {
this.mqMessageSender = mqMessageSender;
this.radarImagePgSqlMapper = radarImagePgSqlMapper;
this.radarImageMapper = radarImageMapper;
this.radarImage = radarImage;
}
Log log = LogFactory.getLog(SyncCmadaasRadaImgReceiver.class);
FtpHelper ftpHelperUpload;
@RabbitListener(queuesToDeclare = {
@Queue(name = "sxqxgxw_sync_cmadaas_radar_img")
})
@RabbitHandler
@Override
public void onMessageReceived(String mqMessageString) {
JsonNode jsonNode = JsonConverter.jsonString2JsonNode(mqMessageString);
JsonNode msg = jsonNode.findValue("msg");
JsonNode JsonNodeParams = msg.findValue("params");
Map<String, Object> params = JsonConverter.jsonNode2HashMap(JsonNodeParams);
if (params.size() > 0) {
String server = params.get("server").toString();
int port = Integer.parseInt(params.get("port").toString());
String username = params.get("username").toString();
String passwd = params.get("passwd").toString();
String startTime = params.get("startTime").toString();
String endTime = params.get("endTime").toString();
//"/static/resource/radar/bexa_cr_nul_1km_png/"
String remoteFtpPath = params.get("remoteFtpPath").toString();
List<RadaBexaCrNul1kmPngRetData.RadaBexaCrNul1kmPngRetDataItem> radaBexaCrNul1kmPngRetDataItemList = radarImage.getRadaBexaCrNul1kmPng(startTime, endTime);
try {
ftpHelperUpload = FtpHelper.getInstance(server, port, username, passwd, "instance3");
boolean connRet1 = ftpHelperUpload.connect();
if (!connRet1) {
log.error(server + " : " + port +" connect ftp failed");
return;
}
if (radaBexaCrNul1kmPngRetDataItemList != null && radaBexaCrNul1kmPngRetDataItemList.size() > 0) {
List<RadarImageItem> radarImageItemPgSqlList = new ArrayList<>();
//读取指定目录下的文件名
List<String> fileList = ftpHelperUpload.getFileList(remoteFtpPath, null);
for (RadaBexaCrNul1kmPngRetData.RadaBexaCrNul1kmPngRetDataItem item : radaBexaCrNul1kmPngRetDataItemList) {
String fileUrl = item.getFileUrl();
String fileName = item.getFileName();
int fileSize = Integer.parseInt(item.getFileSize());
boolean res = false;
if (fileSize > 0) {
//远程目录不包含file时
if (fileList != null && !fileList.contains(fileName)) {
res = ftpHelperUpload.uploadStreamFile(fileUrl, remoteFtpPath, fileName);
}
}
if (res) {
//Z_RADA_C_BEXA_20230724110000_P_DOR_RDCP_CR_NUL_20230724110947.jpg
String datetime = item.getDatetime();
SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmmss");
Date date = sdf.parse(datetime);
RadarImageItem radarImageItem = new RadarImageItem();
radarImageItem.setTime(date);
radarImageItem.setDegree(0);
radarImageItem.setNumber(0);
radarImageItem.setStation("290");
radarImageItem.setPath("http://121.40.189.20:5000/resource/radar/bexa_cr_nul_1km_png/" + fileName);
radarImageItem.setFilename(fileName);
radarImageItem.setProduct("map");
radarImageItem.setExtent("[104.01,31.1,113,40]");
radarImageItemPgSqlList.add(radarImageItem);
}
}
//pgsql插入数据库
if (radarImageItemPgSqlList.size() > 0) {
for (List<RadarImageItem> radarImageItems : ListUtils.splitList(radarImageItemPgSqlList, 10)) {
int rows = radarImagePgSqlMapper.insertIntoRadarImage(radarImageItems);
Thread.sleep(50);
}
Thread.sleep(1000);
}
}
} catch (Exception ex) {
log.error(MessageUtils.getErrMsg(ex));
} finally {
if (ftpHelperUpload.connect()) {
ftpHelperUpload.close();
}
}
}
}
}
FtpHelper工具类，生成实例，本地图片上传，使用图片url打开流上传，下载，查看目标路径下文件
package com.sxqx.utils.ftp;
import com.sxqx.utils.exception.MessageUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.net.ftp.*;
import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
public class FtpHelper {
private final FTPClient ftp;
private static String _ftpServer;
private static int _ftpPort;
private static String _ftpUsername;
private static String _ftpPassword;
private FTPClientStatus status;
private static FtpHelper instance;
Log log = LogFactory.getLog(FtpHelper.class);
public static FtpHelper getInstance(String ftpServer, int ftpPort, String ftpUsername, String ftpPassword) {
if (null == instance) {
_ftpServer = ftpServer;
_ftpPort = ftpPort;
_ftpUsername = ftpUsername;
_ftpPassword = ftpPassword;
instance = new FtpHelper();
}
return instance;
}
public FtpHelper() {
ftp = new FTPClient();
status = FTPClientStatus.DISCONNECTED;
}
/**
* ftp连接服务器
*/
public boolean connect() {
try {
ftp.setDataTimeout(10000);
ftp.setConnectTimeout(10000);
ftp.connect(_ftpServer, _ftpPort);
ftp.login(_ftpUsername, _ftpPassword);
if (FTPReply.isPositiveCompletion(ftp.getReplyCode())) {
status = FTPClientStatus.CONNECTED;
return true;
} else {
status = FTPClientStatus.CONNECT_EXCEPTION;
return false;
}
} catch (Exception e) {
log.error(MessageUtils.getErrMsg(e));
status = FTPClientStatus.CONNECT_EXCEPTION;
return false;
}
}
/**
* 读取指定目录下的文件名
*
* @param path 路径名称
*/
public List<String> getFileList(String path, FTPFileFilter filter) {
List<String> fileLists = new ArrayList<>();
FTPFile[] ftpFiles = null;
if (status == FTPClientStatus.CONNECTED) {
try {
String remotePath = ftp.printWorkingDirectory();
if (!Objects.equals(path, remotePath)) {
ftp.changeWorkingDirectory(path);
}
ftp.enterLocalPassiveMode();
if (filter != null) {
ftpFiles = ftp.listFiles(path, filter);
} else {
ftpFiles = ftp.listFiles(path);
}
} catch (IOException e) {
//e.printStackTrace();
log.error(MessageUtils.getErrMsg(e));
}
for (int i = 0; ftpFiles != null && i < ftpFiles.length; i++) {
FTPFile file = ftpFiles[i];
if (file.isFile()) {
fileLists.add(file.getName());
} else if (file.isDirectory()) {
List<String> childLists = getFileList(path + file.getName() + "/", filter);
fileLists.addAll(childLists);
}
}
}
return fileLists;
}
/**
* 关闭连接
*/
public void close(){
try {
if(ftp.isConnected()){
ftp.logout();
ftp.disconnect();
}
}catch (IOException e){
e.printStackTrace();
}
}
/**
* 根据天擎返回值url作为流上传图片到远程服务器
* @param url
* @param remoteFilePath
* @param file
* @return
*/
public boolean uploadStreamFile(String url, String remoteFilePath, String file) {
try {
ftp.setFileType(ftp.BINARY_FILE_TYPE);
ftp.enterLocalPassiveMode();
String remotePath = ftp.printWorkingDirectory();
if (!Objects.equals(remoteFilePath, remotePath)) {
ftp.changeWorkingDirectory(remoteFilePath);
}
log.info(remoteFilePath);
file = new String(file.getBytes("GBK"), "iso-8859-1");
String name = new File(file).getName();
// 从URL获取图片流
URL imageUrl = new URL(url);
InputStream inputStream = new BufferedInputStream(imageUrl.openStream());
// 上传文件到远程服务器
boolean uploaded = ftp.storeFile(name, inputStream);
inputStream.close();
if (uploaded) {
return true;
} else {
log.info("失败，服务器返回:" + ftp.getReplyString());//获取上传失败的原因
return false;
}
} catch (IOException e) {
e.printStackTrace();
return false;
}
}
}
RadarImageFilter实现类过滤ftp文件
RadarImageFilter实现了FTPFileFilter接口
调用listFiles(final String pathname, final FTPFileFilter filter)方法用于过滤获取到的ftp服务器指定路径下的文件列表
以下RadarImageFilter用于过滤获取到的文件为与当前时间相比，一小时内的文件
package com.sxqx.utils.filter.ftp;
import org.apache.commons.net.ftp.FTPFile;
import org.apache.commons.net.ftp.FTPFileFilter;
import java.text.SimpleDateFormat;
import java.time.Duration;
import java.time.Instant;
import java.util.Calendar;
import java.util.Date;
public class RadarImageFilter implements FTPFileFilter {
@Override
public boolean accept(FTPFile ftpFile) {
if (!ftpFile.isFile()) {
return false;
}
String fileName = ftpFile.getName();
String[] s = fileName.split("_");
String dt = s[1].substring(0, 14);
SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmmss");
try {
Date date = sdf.parse(dt);
Calendar bjc = Calendar.getInstance();
bjc.setTime(date);
bjc.add(Calendar.HOUR_OF_DAY, 8);
Date calTime = bjc.getTime();
Date now = new Date();
Instant startInstant = calTime.toInstant();
Instant endInstant = now.toInstant();
// 使用Duration.between()方法计算两个Instant对象之间的持续时间
Duration duration = Duration.between(startInstant, endInstant);
// 调用Duration对象的toHours()方法，以获取两个日期之间的小时差
long diffHours = Math.abs(duration.toHours());
return diffHours < 1;
} catch (Exception ex) {
ex.printStackTrace();
return false;
}
}
}
application.yml中配置了多个数据源和mq配置
spring:
servlet:
multipart:
max-file-size: 300MB
max-request-size: 300MB
datasource:
remote-xugu1:
username: username
password: password
driver-class-name: com.xugu.cloudjdbc.Driver
url: jdbc:xugu://xugu_ip:33000/SYSTEM
pgsql:
username: username
password: password
driver-class-name: org.postgresql.Driver
url: jdbc:postgresql://pgsql_ip:5432/yunmei
rabbitmq:
username: guest
password: guest
virtual-host: /
publisher-returns: true
listener:
simple:
acknowledge-mode: auto # 手动应答
prefetch: 10 #每次从队列中取一个,轮询分发，默认是公平分发
retry:
max-attempts: 5 # 重试次数
enabled: true # 开启重试
concurrency: 5
max-concurrency: 10
host: mq_ip # ip
port: 5672
publisher-confirm-type: simple
logging:
level:
org.mybatis.spring: OFF
java.sql: OFF
org.springframework.web: OFF
com.sxqx.mapper: OFF
pgsql数据源配置文件
@ConfigurationProperties(prefix = “spring.datasource.pgsql”)对应application.yml中层级关系
@MapperScan(basePackages = “com.sxqx.mapper.remote.pgsql”)对应mybatis包扫描路径
"classpath:mapper/remote/pgsql//.xml"对应xml文件路径
package com.sxqx.config.datasource;
import org.apache.ibatis.session.SqlSessionFactory;
import org.mybatis.spring.SqlSessionFactoryBean;
import org.mybatis.spring.SqlSessionTemplate;
import org.mybatis.spring.annotation.MapperScan;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.autoconfigure.jdbc.DataSourceProperties;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
import javax.sql.DataSource;
@Configuration
@MapperScan(basePackages = "com.sxqx.mapper.remote.pgsql", sqlSessionFactoryRef = "remotePgSqlSqlSessionFactory")
public class RemotePgSqlDatasourceConfig {
@Bean(name = "remotePgSqlDatasourceProperties")
@ConfigurationProperties(prefix = "spring.datasource.pgsql")
public DataSourceProperties remoteXuGu1DatasourceProperties(){
return new DataSourceProperties();
}
@Bean(name = "remotePgSqlDatasource")
public DataSource remotePgSqlDatasource(@Qualifier("remotePgSqlDatasourceProperties") DataSourceProperties dataSourceProperties) {
return dataSourceProperties.initializeDataSourceBuilder().build();
}
@Bean("remotePgSqlSqlSessionFactory")
public SqlSessionFactory remotePgSqlSqlSessionFactory(@Qualifier("remotePgSqlDatasource") DataSource dataSource) throws Exception {
SqlSessionFactoryBean bean = new SqlSessionFactoryBean();
bean.setDataSource(dataSource);
bean.setMapperLocations(new PathMatchingResourcePatternResolver().getResources("classpath:mapper/remote/pgsql/*/*.xml"));
return bean.getObject();
}
@Bean("remotePgSqlSessionTemplate")
public SqlSessionTemplate remoteXuGu1SqlSessionTemplate(@Qualifier("remotePgSqlSqlSessionFactory") SqlSessionFactory sqlSessionFactory) {
return new SqlSessionTemplate(sqlSessionFactory);
}
}
虚谷数据源配置类
虚谷数据源数据源配置类似于上述pgsql
package com.sxqx.config.datasource;
import org.apache.ibatis.session.SqlSessionFactory;
import org.mybatis.spring.SqlSessionFactoryBean;
import org.mybatis.spring.SqlSessionTemplate;
import org.mybatis.spring.annotation.MapperScan;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.autoconfigure.jdbc.DataSourceProperties;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
import javax.sql.DataSource;
@Configuration
@MapperScan(basePackages = "com.sxqx.mapper.remote.xugu1", sqlSessionFactoryRef = "remoteXuGu1SqlSessionFactory")
public class RemoteXuGu1DatasourceConfig {
@Bean(name = "remoteXuGu1DatasourceProperties")
@ConfigurationProperties(prefix = "spring.datasource.remote-xugu1")
public DataSourceProperties remoteXuGu1DatasourceProperties(){
return new DataSourceProperties();
}
@Bean(name = "remoteXuGu1Datasource")
public DataSource remoteXuGu1Datasource(@Qualifier("remoteXuGu1DatasourceProperties") DataSourceProperties dataSourceProperties) {
return dataSourceProperties.initializeDataSourceBuilder().build();
}
@Bean("remoteXuGu1SqlSessionFactory")
public SqlSessionFactory remoteXuGu1SqlSessionFactory(@Qualifier("remoteXuGu1Datasource") DataSource dataSource) throws Exception {
SqlSessionFactoryBean bean = new SqlSessionFactoryBean();
bean.setDataSource(dataSource);
bean.setMapperLocations(new PathMatchingResourcePatternResolver().getResources("classpath:mapper/remote/xugu1/*/*.xml"));
return bean.getObject();
}
@Bean("remoteXuGu1SqlSessionTemplate")
public SqlSessionTemplate remoteXuGu1SqlSessionTemplate(@Qualifier("remoteXuGu1SqlSessionFactory") SqlSessionFactory sqlSessionFactory) {
return new SqlSessionTemplate(sqlSessionFactory);
}
}
Mapper层
com.sxqx.mapper.remote.pgsql包路径对应配置的，mybatis扫描路径
package com.sxqx.mapper.remote.pgsql.radar;
import com.sxqx.pojo.RadarImageItem;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
import java.util.List;
@Mapper
public interface IRadarImagePgSqlMapper {
int insertIntoRadarImage(@Param("radarImageItems") List<RadarImageItem> radarImageItems);
}
pgsql xml文件
与虚谷数据库相比，批量插入时，需要使用",“逗号分隔，separator=”,"
<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE mapper
PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
"http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.sxqx.mapper.remote.pgsql.radar.IRadarImagePgSqlMapper">
<resultMap id="RadarImageItem" type="com.sxqx.pojo.RadarImageItem">
<result column="Id" property="id"/>
<result column="time" property="time"/>
<result column="degree" property="degree"/>
<result column="no" property="number"/>
<result column="station" property="station"/>
<result column="path" property="path"/>
<result column="filename" property="filename"/>
<result column="product" property="product"/>
</resultMap>
<insert id="insertIntoRadarImage"
parameterType="java.util.List"
>
INSERT INTO radar_img
(
time,
degree,
no,
station,
path,
filename,
product
)
VALUES
<foreach collection="radarImageItems" item="radarImageItem" separator=",">
(
#{radarImageItem.time},
#{radarImageItem.degree},
#{radarImageItem.number},
#{radarImageItem.station},
#{radarImageItem.path},
#{radarImageItem.filename},
#{radarImageItem.product}
)
</foreach>
</insert>
</mapper>
pom文件引入依赖包
<!--ftp-->
<dependency>
<groupId>commons-net</groupId>
<artifactId>commons-net</artifactId>
<version>3.8.0</version>
</dependency>
<!--pgsql数据库-->
<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
<scope>runtime</scope>
</dependency>
centos安装ftp命令
centos安装ftp
yum install -y vsftpd
关闭匿名访问
vi /etc/vsftpd/vsftpd.conf
改anonymous_enable=no
创建ftp用户
useradd -s /sbin/nologin ftpuser
设置密码
passwd ftpuser
ftpuser
设置ftpuser只能连接ftp
vi /etc/shells
/sbin/nologin
启动服务
systemctl start vsftpd
systemctl status vsftpd
开启ftp开机启动
systemctl enable vsftpd
防火墙永久开启ftp
firewall-cmd --zone=public --add-service=ftp --permanent
重启防火墙
systemctl restart firewalld
ftp远程a读取文件，远程b下载文件
mq消费端
import com.fasterxml.jackson.databind.JsonNode;
import com.sxqx.entity.weatherlivepic.WeatherLivePicEntity;
import com.sxqx.listener.IMessageReceiver;
import com.sxqx.mapper.remote.xugugzb.weatherlivepic.IWeatherLivePicMapper;
import com.sxqx.utils.common.ListUtils;
import com.sxqx.utils.dataConverter.JsonConverter;
import com.sxqx.utils.exception.MessageUtils;
import com.sxqx.utils.filter.ftp.WeatherLivePicFilter;
import com.sxqx.utils.ftp.FtpHelper;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.net.ftp.FTPFileFilter;
import org.springframework.amqp.rabbit.annotation.Queue;
import org.springframework.amqp.rabbit.annotation.RabbitHandler;
import org.springframework.amqp.rabbit.annotation.RabbitListener;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.*;
/**
* 同步16.123上生成的色斑图pic到60
*/
@Component
public class SyncWeatherLivePicMessageReceiver implements IMessageReceiver {
private final IWeatherLivePicMapper weatherLivePicMapper;
@Autowired
public SyncWeatherLivePicMessageReceiver(IWeatherLivePicMapper weatherLivePicMapper) {
this.weatherLivePicMapper = weatherLivePicMapper;
}
Log log = LogFactory.getLog(SyncWeatherLivePicMessageReceiver.class);
private FtpHelper ftpHelperIns1;
private FtpHelper ftpHelperIns2;
private FTPFileFilter ftpFileFilter;
@RabbitListener(queuesToDeclare = {
@Queue(name = "sxqxgzbgxw_weather_live_pic")
})
@RabbitHandler
@Override
public void onMessageReceived(String mqMessageString) {
JsonNode jsonNode = JsonConverter.jsonString2JsonNode(mqMessageString);
JsonNode msg = jsonNode.findValue("msg");
JsonNode JsonNodeParams = msg.findValue("params");
Map<String, Object> params = JsonConverter.jsonNode2HashMap(JsonNodeParams);
if (params.size() > 0) {
String times = params.get("times").toString();
String serverFrom = params.get("serverFrom").toString();
int portFrom = Integer.parseInt(params.get("portFrom").toString());
String usernameFrom = params.get("usernameFrom").toString();
String passwdFrom = params.get("passwdFrom").toString();
String serverTo = params.get("serverTo").toString();
int portTo = Integer.parseInt(params.get("portTo").toString());
String usernameTo = params.get("usernameTo").toString();
String passwdTo = params.get("passwdTo").toString();
Map<String,String> map = new HashMap<>();
map.put("PRE_12h","过去12小时降水量");
map.put("PRE_1h","过去1小时降水量");
map.put("PRE_24h","过去24小时降水量");
map.put("PRE_3h","过去3小时降水量");
map.put("PRE_6h","过去6小时降水量");
map.put("RHU","相对湿度");
map.put("RHU_Min","最小相对湿度");
map.put("Snow_Depth","积雪深度");
map.put("TEM","温度");
map.put("TEM_ChANGE_24h","24小时温度变化");
map.put("TEM_Max","最高气温");
map.put("TEM_Max_24h","24小时最高气温");
map.put("TEM_Min","最低气温");
map.put("TEM_Min_24h","24小时最低气温");
map.put("VIS","能见度");
map.put("VIS_HOR_10MI","10分钟平均水平能见度");
map.put("VIS_HOR_1MI","1分钟平均水平能见度");
map.put("VIS_Min","最小水平能见度");
map.put("WIN_S_Avg_10mi","10分钟平均风速");
map.put("WIN_S_Avg_2mi","2分钟平均风速");
map.put("WIN_S_Inst_Max","极大风速");
map.put("WIN_S_Max","最大风速");
Set<String> types = map.keySet();
List<WeatherLivePicEntity> livePicEntities = new ArrayList<>();
try {
//16.123下载图片流
ftpHelperIns1 = FtpHelper.getInstance(serverFrom, portFrom, usernameFrom, passwdFrom, "instance1");
boolean connIns1 = ftpHelperIns1.connect();
if (!connIns1) {
return;
}
//60.204.202.112上传图片流
ftpHelperIns2 = FtpHelper.getInstance(serverTo, portTo, usernameTo, passwdTo, "instance2");
boolean connIns2 = ftpHelperIns2.connect();
if (!connIns2) {
return;
}
for (String type : types) {
String remoteFtpPathFrom = params.get("remoteFtpPathFrom").toString()+ times +"/"+ type + "/" + "gjzjqyz";
String remoteFtpPathTo = params.get("remoteFtpPathTo").toString()+ times +"/"+ type + "/" + "gjzjqyz";
//判断目标服务器是否有此目录，没有就创建
ftpHelperIns2.makeDirectory(remoteFtpPathTo);
//目标服务器此路径文件
List<String> fileListTo = ftpHelperIns2.getFileList(remoteFtpPathTo,ftpFileFilter);
if(fileListTo != null || fileListTo.size()>0){
continue;
}
//源服务器此路径是否有文件
ftpFileFilter = new WeatherLivePicFilter();
List<String> fileListFrom = ftpHelperIns1.getFileList(remoteFtpPathFrom,ftpFileFilter);
if(fileListFrom==null || fileListFrom.size()==0){
continue;
}
for(String fileNameFrom : fileListFrom){
//如果源服务器有png
if(fileNameFrom.endsWith(".png")){
//如果目标路径没有png
if(fileListTo==null || fileListTo.size()==0 || !fileListTo.contains(fileNameFrom)){
byte[] fileData = ftpHelperIns1.getFileData(remoteFtpPathFrom, fileNameFrom);
boolean res = ftpHelperIns2.uploadStreamFile(fileData, remoteFtpPathTo, fileNameFrom);
if (res) {
WeatherLivePicEntity weatherLivePicEntity = new WeatherLivePicEntity();
weatherLivePicEntity.setType("地面数据");
weatherLivePicEntity.setPicName(fileNameFrom);
weatherLivePicEntity.setPicUrl("http://远程b/dataSharingStatic/weatherlive/colorFigure/png/610000/610000/"+ times +"/"+ type + "/" + "gjzjqyz/"+ fileNameFrom);
//weatherLivePicEntity.setPicExtent("");
weatherLivePicEntity.setElementCode(type);
weatherLivePicEntity.setElementName(map.get(type));
DateFormat dateFormat = new SimpleDateFormat("yyyyMMddHHmmss");
Date date = dateFormat.parse(times);
// 创建一个Calendar对象，用于进行时间计算
Calendar calendar = Calendar.getInstance();
calendar.setTime(date);
// 将时间+8小时
calendar.add(Calendar.HOUR_OF_DAY, +8);
date = calendar.getTime();
weatherLivePicEntity.setDateTime(date);
livePicEntities.add(weatherLivePicEntity);
}
}
}
}
}
}catch (Exception ex) {
log.error(MessageUtils.getErrMsg(ex));
} finally {
//关闭资源
if (ftpHelperIns1.connect()) {
ftpHelperIns1.close();
}
if (ftpHelperIns2.connect()) {
ftpHelperIns2.close();
}
}
if (livePicEntities.size() > 0) {
for (List<WeatherLivePicEntity> weatherLivePicEntities : ListUtils.splitList(livePicEntities, 10)) {
weatherLivePicMapper.insertWeatherLivePic(weatherLivePicEntities);
try {
Thread.sleep(50);
} catch (InterruptedException ex) {
//ex.printStackTrace();
log.error(MessageUtils.getErrMsg(ex));
}
}
}
}
}
}
过滤器过滤文件
import org.apache.commons.net.ftp.FTPFile;
import org.apache.commons.net.ftp.FTPFileFilter;
public class WeatherLivePicFilter implements FTPFileFilter {
@Override
public boolean accept(FTPFile ftpFile) {
boolean flag = true;
if (!ftpFile.isFile()) {
flag = false;
return flag;
}
String fileName = ftpFile.getName();
if (!fileName.endsWith(".png")) {
flag = false;
return flag;
}
return flag;
}
}
ftp工具类
package com.sxqx.utils.ftp;
import com.sxqx.utils.StringUtils;
import com.sxqx.utils.exception.MessageUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.net.ftp.*;
import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.*;
import java.net.URL;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
public class FtpHelper {
private final FTPClient ftp;
private static String _ftpServer;
private static int _ftpPort;
private static String _ftpUsername;
private static String _ftpPassword;
private FTPClientStatus status;
private static FtpHelper instance;
private static String instanceName;
Log log = LogFactory.getLog(FtpHelper.class);
public static FtpHelper getInstance(String ftpServer, int ftpPort, String ftpUsername, String ftpPassword) {
if (null == instance) {
_ftpServer = ftpServer;
_ftpPort = ftpPort;
_ftpUsername = ftpUsername;
_ftpPassword = ftpPassword;
instance = new FtpHelper();
}
return instance;
}
// 重载的getInstance方法，用于生成不同的实例
public static FtpHelper getInstance(String ftpServer, int ftpPort, String ftpUsername, String ftpPassword, String instanceName) {
if (null == instance || !instanceName.equals(instance.getInstanceName())) {
_ftpServer = ftpServer;
_ftpPort = ftpPort;
_ftpUsername = ftpUsername;
_ftpPassword = ftpPassword;
instance = new FtpHelper();
instance.setInstanceName(instanceName);
}
return instance;
}
// 设置实例名称的方法
public void setInstanceName(String name) {
this.instanceName = name;
}
// 获取实例名称的方法
public String getInstanceName() {
return this.instanceName;
}
/**
*
*/
public FtpHelper() {
ftp = new FTPClient();
status = FTPClientStatus.DISCONNECTED;
}
/**
* ftp连接服务器
*/
public boolean connect() {
try {
ftp.setDataTimeout(10000);
ftp.setConnectTimeout(10000);
ftp.connect(_ftpServer, _ftpPort);
ftp.login(_ftpUsername, _ftpPassword);
if (FTPReply.isPositiveCompletion(ftp.getReplyCode())) {
status = FTPClientStatus.CONNECTED;
return true;
} else {
status = FTPClientStatus.CONNECT_EXCEPTION;
return false;
}
} catch (Exception e) {
log.error(MessageUtils.getErrMsg(e));
status = FTPClientStatus.CONNECT_EXCEPTION;
return false;
}
}
/**
* 读取指定目录下的文件名
*
* @param path 路径名称
*/
public List<String> getFileList(String path, FTPFileFilter filter) {
List<String> fileLists = new ArrayList<>();
FTPFile[] ftpFiles = null;
if (status == FTPClientStatus.CONNECTED) {
try {
ftp.enterLocalPassiveMode();
String remotePath = ftp.printWorkingDirectory();
if (!Objects.equals(path, remotePath)) {
ftp.changeWorkingDirectory(path);
}
remotePath = ftp.printWorkingDirectory();
if (filter != null) {
ftpFiles = ftp.listFiles(path, filter);
} else {
ftpFiles = ftp.listFiles(path);
}
} catch (IOException e) {
//e.printStackTrace();
log.error("失败，服务器返回:" + ftp.getReplyString());//获取上传失败的原因
log.error(MessageUtils.getErrMsg(e));
}
for (int i = 0; ftpFiles != null && i < ftpFiles.length; i++) {
FTPFile file = ftpFiles[i];
if (file.isFile()) {
fileLists.add(file.getName());
} else if (file.isDirectory()) {
List<String> childLists = getFileList(path + file.getName() + "/", filter);
fileLists.addAll(childLists);
}
}
}
return fileLists;
}
/**
* 关闭连接
*/
public void close() {
try {
if (ftp.isConnected()) {
ftp.logout();
ftp.disconnect();
}
} catch (IOException e) {
//e.printStackTrace();
log.error(MessageUtils.getErrMsg(e));
}
}
/**
* 读取远程服务器文件到内存,返回字节数组
*
* @param ftpPath FTP路径
* @param fileName 文件名
*/
public byte[] getFileData(String ftpPath, String fileName) {
ByteArrayOutputStream outputStream = null;
byte[] fileData = null;
if (status == FTPClientStatus.CONNECTED) {
try {
ftp.setFileType(FTP.BINARY_FILE_TYPE);
ftp.enterLocalPassiveMode();
String remotePath = ftp.printWorkingDirectory();
if (!Objects.equals(ftpPath, remotePath)) {
ftp.changeWorkingDirectory(ftpPath);
}
// 从远程服务器A读取文件内容到内存
outputStream = new ByteArrayOutputStream();
if (ftp.retrieveFile(fileName, outputStream)) {
fileData = outputStream.toByteArray();
} else {
log.error("文件下载失败: " + fileName);
}
} catch (IOException e) {
log.error(MessageUtils.getErrMsg(e));
} finally {
try {
if(outputStream!=null){
outputStream.close();
}
} catch (IOException e) {
log.error("关闭输出流出错: " + e.getMessage());
}
}
}
return fileData;
}
/**
* 上传流文件到指定目录
*
* @return
*/
public boolean uploadStreamFile(byte[] fileData, String remoteFilePath, String file) {
ByteArrayInputStream inputStream = null;
try {
ftp.setFileType(ftp.BINARY_FILE_TYPE);
ftp.enterLocalPassiveMode();
String remotePath = ftp.printWorkingDirectory();
if (!Objects.equals(remoteFilePath, remotePath)) {
ftp.changeWorkingDirectory(remoteFilePath);
}
remotePath = ftp.printWorkingDirectory();
// 将文件内容从内存上传到远程服务器B
inputStream = new ByteArrayInputStream(fileData);
// 上传文件到远程服务器
String name = new File(file).getName();
boolean uploaded = ftp.storeFile(name, inputStream);
if (uploaded) {
return true;
} else {
log.info("失败，服务器返回:" + ftp.getReplyString());//获取上传失败的原因
return false;
}
} catch (IOException e) {
return false;
} finally {
try {
if (inputStream!=null){
inputStream.close();
}
} catch (IOException e) {
log.error("关闭输出流出错: " + e.getMessage());
}
}
}
/**
* 创建多级目录
*
* @return
*/
public boolean makeDirectory(String createPath) {
boolean makeDirectory = false;
try {
ftp.enterLocalPassiveMode();
ftp.changeWorkingDirectory("/");
String[] split = createPath.split("/");
for (String str : split) {
if(StringUtils.isEmpty(str)){
continue;
}
if (!ftp.changeWorkingDirectory(str)) {
makeDirectory = ftp.makeDirectory(str);
ftp.changeWorkingDirectory(str);
}
}
} catch (IOException e) {
return false;
}
return makeDirectory;
}
}
ftp上传下载时有时候会出现拒绝连接错误
2023-10-17 00:10:15.599 ERROR com.sxqx.listener.impl.SyncWeatherLivePicMessageReceiver Line:154-java.net.ConnectException: 拒绝连接 (Connection refused)
at java.net.PlainSocketImpl.socketConnect(Native Method)
at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:476)
at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:218)
at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:200)
at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:394)
at java.net.Socket.connect(Socket.java:606)
at org.apache.commons.net.ftp.FTPClient._openDataConnection_(FTPClient.java:866)
at org.apache.commons.net.ftp.FTPClient._storeFile(FTPClient.java:1053)
at org.apache.commons.net.ftp.FTPClient.storeFile(FTPClient.java:3816)
at org.apache.commons.net.ftp.FTPClient.storeFile(FTPClient.java:3846)
at com.sxqx.utils.ftp.FtpHelper.uploadStreamFile(FtpHelper.java:395)
at com.sxqx.listener.impl.SyncWeatherLivePicMessageReceiver.onMessageReceived(SyncWeatherLivePicMessageReceiver.java:128)
解决方案：
FTP被动模式及超时问题解决，FTPClient.listFiles或者FTPClient.retrieveFile方法假死
FTPClient 每隔多少秒发送一次心跳信息
392
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
