背景: 前段时间，在与厂家对接数据的过程中可能需要用到sftp进行数据的交互，它相较于ftp具有比较安全的加密方式、但传输效率略低，因此做了这么一个小工具来进行自动的拉取发送数据文件和sftp交互。
代码结构:
目录中各个类的作用大概如下:
1.SftpOpBean.java:装载配置文件get/put对象
2.SftpUpDoException.java:自定义异常，没啥特殊用处
3.GetFileFromSftpTask.java:从sftp拉取文件的任务(线程)
4.PutFileToSftpTask.java:将本地文件发送到sftp的任务(线程)
5.SftpUtils.java:sftp操作的工具类，主要是获取sftp对象及常用方法
6.ToolUtil.java:常用小工具封装，如获取文件大小、执行shell命令等
7.MainConstance.java:常量定义
8.MainTask.java:主程序启动类
依赖lib包:
模糊的那个jar在此处主要是提供XML读取的工具类，因公司内部封装特此加码。
配置及程序:
config.xml
<?xml version="1.0" encoding="UTF-8"?>
<conf>
<default>
<!-- put节点为将本地文件路径下的文件发送到到sftp远端目录下,src:本地待发送文件路径, dst:sftp上的接收文件路径,user:sftp创建的一个用户,pwd:用户的密码,ip和port为sftp相关属性。put节点可配置多个，同时生效。-->
<put src="./data" dst="upload/" user="sftp" pwd="sftp123" ip="24.22.10.11" port="22" />
<!--get节点为将sftp远端目录下的数据文件拉取到本地,src:sftp上的源文件路径,dst:本地接收目的文件路径, 同理get节点也可以配置多个-->
<get src="down/" dst="./get" user="sftp" pwd="sftp123" ip="24.22.10.11" port="22"/>
</default>
</conf>
MainTask.java
public class MainTask {
public static void main(String[] args) {
logger.info("任务启动开始......");
init();
// TODO 启动PUT线程
if (CollectionUtils.isNotEmpty(MainConstance.PUT_CONTAINER) ) {
ScheduledExecutorService putThreadPool = Executors.newScheduledThreadPool(MainConstance.PUT_CONTAINER.size());
for (SftpOpBean bean : MainConstance.PUT_CONTAINER) {
putThreadPool.execute(new PutFileToSftpTask("【PutFileToFtp:"+bean.getIp()+"】", bean));
}
}
// TODO 启动GET线程
if (CollectionUtils.isNotEmpty(MainConstance.GET_CONTAINER) ) {
ScheduledExecutorService putThreadPool = Executors.newScheduledThreadPool(MainConstance.GET_CONTAINER.size());
for (SftpOpBean bean : MainConstance.GET_CONTAINER) {
putThreadPool.execute(new GetFileFromSftpTask("【GetFileFromSftp:"+bean.getIp()+"】", bean));
}
}
}
/**
* @Description 初始化
*/
private static void init() {
logger.info("初始化开始......");
// 初始化常量池
try {
// 读取配置文件
MainConstance.init();
}catch (Exception e){
logger.error("初始化发生异常:",e);
}
logger.info("初始化完成......");
}
}
PutFileToSftpTask.java
public class PutFileToSftpTask implements Runnable {
private static final Logger logger =Logger.getLogger("LOGPUT");
// 监控文件夹对象
private WatchService watchService;
private boolean notDone = true;
private String module;
// sftp发送拉取配置对象
private SftpOpBean bean;
// sftp操作工具类
private SftpUtils sftpUtils;
public PutFileToSftpTask(String module, SftpOpBean bean) {
this.module=module;
this.bean=bean;
this.sftpUtils=new SftpUtils(bean.getUserName(),bean.getPwd(),bean.getIp(),bean.getPort());
}
public void run() {
logger.info(module+" 开始任务[PutFileToSftpTask]......");
try {
process();
} catch (Exception e) {
logger.error(module+" PutFileToSftpTask run failed." + e.getMessage(), e);
}
}
/**
* @Description：监控输入目录
* @param inPath
* @throws SftpUpDoException
*/
private void init(String inPath) throws SftpUpDoException {
Path path = Paths.get(inPath);
try {
watchService = FileSystems.getDefault().newWatchService();
// 注册监控对象监控事件:文件创建、文件删除、修改
path.register(watchService, StandardWatchEventKinds.ENTRY_CREATE, StandardWatchEventKinds.ENTRY_DELETE,
StandardWatchEventKinds.ENTRY_MODIFY);
} catch (Exception e) {
throw new SftpUpDoException(module+" PutFileToSftpTask init failed. --" + e.getMessage(), e);
}
}
private void process() throws SftpUpDoException {
init(bean.getSrc());
logger.info(module+" 开始任务监控输入目录[" + bean.getSrc() + "]......");
while (notDone) {
try {
WatchKey watchKey = this.watchService.take();
if (watchKey != null) {
List<WatchEvent<?>> events = watchKey.pollEvents();
for (WatchEvent event : events) {
WatchEvent.Kind<?> kind = event.kind();
if (kind != StandardWatchEventKinds.OVERFLOW) {
WatchEvent<Path> ev = event;
Path path = (Path) ev.context();
// 检测到有文件创建
if (kind == StandardWatchEventKinds.ENTRY_CREATE) {
String fileName = path.getFileName().toString();
// 并且文件创建完成(大小不变)
if (ToolUtil.fileIsCreateSuccess(bean.getSrc()+"/"+fileName)) {
logger.info(module+" <<<<<<发现文件["+bean.getSrc()+"] :" + fileName
+",大小:"+ToolUtil.getFileSize(bean.getSrc()+"/"+fileName));
putFile(fileName);
}
}
}
}
if (!watchKey.reset()) {
logger.info(module+" exit watch server");
break;
}
}
} catch (Exception e) {
logger.error(module+" PutFileToSftpTask watch start error:" + e.getMessage(), e);
throw new SftpUpDoException(module+" PutFileToSftpTask watch start error:" + e.getMessage(), e);
}
}
}
/**
* @Description put文件
*/
private void putFile(String fileName) {
try {
logger.info(module+" <<<<<<处理文件["+bean.getSrc()+"] :" + fileName);
// 发送文件
sftpUtils.uploadFile(bean.getDst(),bean.getSrc()+"/"+fileName,fileName);
// 删除本地源文件
ToolUtil.deleteFile(bean.getSrc()+"/"+fileName);
logger.info(module+" <<<<<<处理文件完成!!["+bean.getSrc()+"] :" + fileName);
}catch (Exception e){
logger.error(module+" PutFileToSftpTask error:" + e.getMessage(), e);
}
}
}
GetFileFromSftpTask.java
public class GetFileFromSftpTask implements Runnable {
private static final Logger logger = Logger.getLogger("LOGGET");
private boolean notDone = true;
private String module;
private SftpOpBean bean;
private SftpUtils sftpUtils;
public GetFileFromSftpTask(String module, SftpOpBean bean) {
this.module=module;
this.bean=bean;
this.sftpUtils=new SftpUtils(bean.getUserName(),bean.getPwd(),bean.getIp(),bean.getPort());
}
public void run() {
logger.info(module+" 开始任务[GetFileFromSftpTask]......");
try {
process();
} catch (Exception e) {
logger.error(module+" GetFileFromSftpTask run failed." + e.getMessage(), e);
}
}
private void process() throws SftpUpDoException {
logger.info(module+" 开始任务监控输入目录[" + bean.getSrc() + "]......");
while (notDone) {
try {
// 获取sftp上特定目录下的所有文件
List<String> data=sftpUtils.listFiles(bean.getSrc());
if (CollectionUtils.isNotEmpty(data)){
for (String fileName:
data) {
logger.info(module+" <<<<<<处理文件["+bean.getSrc()+"] :" + fileName);
String dstFile=bean.getDst().endsWith("/")?bean.getDst()+fileName:bean.getDst()+"/"+fileName;
// 下载sftp上的文件到本地
File f=sftpUtils.downloadFile(bean.getSrc(),fileName,dstFile);
if (f.canRead()){
// 删除sftp上的文件
sftpUtils.deleteFile(bean.getSrc(),fileName);
}
logger.info(module+" <<<<<<处理文件完成!!["+bean.getSrc()+"] :" + fileName);
}
}else {
ToolUtil.doSleep(5);
logger.info(module+" GetFileFromSftpTask --no file wait 5s,now:"+new Date().toLocaleString());
}
} catch (Exception e) {
logger.error(module+" GetFileFromSftpTask watch start error:" + e.getMessage(), e);
throw new SftpUpDoException(module+" GetFileFromSftpTask watch start error:" + e.getMessage(), e);
}
}
}
}
SftpUtils.java
public class SftpUtils {
/**log*/
private static final org.apache.log4j.Logger log = Logger.getLogger("LOGFILE");
public static final String NO_FILE = "No such file";
private ChannelSftp sftp = null;
private Session sshSession = null;
private String username;
private String password;
private String host;
private int port;
public SftpUtils(String username, String password, String host, int port) {
this.username = username;
this.password = password;
this.host = host;
this.port = port;
}
/**
* 连接sftp服务器
* @return ChannelSftp sftp类型
*/
public ChannelSftp connect() throws SftpUpDoException{
//log.info("FtpUtil-->connect--ftp连接开始>>>>>>host=" + host + ">>>port" + port + ">>>username=" + username);
JSch jsch = new JSch();
try {
jsch.getSession(username, host, port);
sshSession = jsch.getSession(username, host, port);
//log.info("ftp---Session created.");
sshSession.setPassword(password);
Properties properties = new Properties();
// 测试时设置 主机公钥确认为no.
properties.put("StrictHostKeyChecking", "no");
sshSession.setConfig(properties);
sshSession.setTimeout(6000);
sshSession.connect();
//log.info("ftp---Session connected.");
Channel channel = sshSession.openChannel("sftp");
channel.connect();
//log.info("Opening Channel.");
sftp = (ChannelSftp) channel;
log.info("ftp---Connected to " + host);
}
catch (JSchException e) {
throw new SftpUpDoException("FtpUtil-->connect异常" ,e);
}
return sftp;
}
/**
* 下载单个文件
* @param directory ：远程下载目录(以路径符号结束)
* @param remoteFileName FTP服务器文件名称 如：xxx.txt ||xxx.txt.zip
* @param localFile 本地文件路径 如 D:\\xxx.txt
* @return
* @throws SftpUpDoException
*/
public File downloadFile(String directory, String remoteFileName,String localFile) throws SftpUpDoException {
log.info(">>>>>>>>FtpUtil-->downloadFile--ftp下载文件"+remoteFileName+"开始>>>>>>>>>>>>>");
connect();
File file = null;
OutputStream output = null;
try {
file = new File(localFile);
if (file.exists()){
file.delete();
}
file.createNewFile();
sftp.cd(directory);
output = new FileOutputStream(file);
sftp.get(remoteFileName, output);
log.info("===DownloadFile:" + remoteFileName + " success from sftp.");
}
catch (SftpException e) {
if (e.toString().equals(NO_FILE)) {
log.info(">>>>>>>>FtpUtil-->downloadFile--ftp下载文件失败" + directory +remoteFileName+ "不存在>>>>>>>>>>>>>");
throw new SftpUpDoException("FtpUtil-->downloadFile--ftp下载文件失败" + directory +remoteFileName + "不存在");
}
throw new SftpUpDoException("ftp目录或者文件异常，检查ftp目录和文件" + e.toString());
}
catch (FileNotFoundException e) {
throw new SftpUpDoException("本地目录异常，请检查" + file.getPath() + e.getMessage());
}
catch (IOException e) {
throw new SftpUpDoException("创建本地文件失败" + file.getPath() + e.getMessage());
}
finally {
if (output != null) {
try {
output.close();
}
catch (IOException e) {
throw new SftpUpDoException("Close stream error."+ e.getMessage());
}
}
disconnect();
}
log.info(">>>>>>>>FtpUtil-->downloadFile--ftp下载文件结束>>>>>>>>>>>>>");
return file;
}
/**
* 上传单个文件
* @param directory ：远程下载目录(以路径符号结束)
* @param uploadFilePath 要上传的文件 如：D:\\test\\xxx.txt
* @param fileName FTP服务器文件名称 如：xxx.txt ||xxx.txt.zip
* @throws SftpUpDoException
*/
public void uploadFile(String directory, String uploadFilePath, String fileName)
throws SftpUpDoException {
log.info(">>>>>>>>FtpUtil-->uploadFile--ftp上传文件开始>>>>>>>>>>>>>");
FileInputStream in = null;
connect();
try {
sftp.cd(directory);
}
catch (SftpException e) {
try {
sftp.mkdir(directory);
sftp.cd(directory);
}
catch (SftpException e1) {
throw new SftpUpDoException("ftp创建文件路径失败，路径为" + directory);
}
}
File file = new File(uploadFilePath);
try {
in = new FileInputStream(file);
sftp.put(in, fileName);
}
catch (FileNotFoundException e) {
throw new SftpUpDoException("文件不存在-->" + uploadFilePath);
}
catch (SftpException e) {
throw new SftpUpDoException("sftp异常-->" + e.getMessage());
}
finally {
if (in != null){
try {
in.close();
}
catch (IOException e) {
throw new SftpUpDoException("Close stream error."+ e.getMessage());
}
}
disconnect();
}
log.info(">>>>>>>>FtpUtil-->uploadFile--ftp上传文件结束>>>>>>>>>>>>>");
}
/**
* 删除单个文件
* @param directory ：远程下载目录(以路径符号结束)
* @param fileName FTP服务器文件名称 如：xxx.txt ||xxx.txt.zip
* @throws SftpUpDoException
*/
public void deleteFile(String directory, String fileName)
throws SftpUpDoException {
log.info(">>>>>>>>FtpUtil-->deleteFile--ftp删除文件>>>>>>>>>>>>>");
connect();
try {
sftp.cd(directory);
}
catch (SftpException e) {
try {
sftp.mkdir(directory);
sftp.cd(directory);
}
catch (SftpException e1) {
throw new SftpUpDoException("ftp创建文件路径失败，路径为" + directory);
}
}
try {
log.info(">>>>>>>>FtpUtil-->deleteFile--ftp删除文件>>>>>>>>>>>>>"+fileName);
sftp.rm(fileName);
}
catch (SftpException e) {
throw new SftpUpDoException("sftp异常-->" + e.getMessage());
}
finally {
disconnect();
}
log.info(">>>>>>>>FtpUtil-->deleteFile--ftp删除文件成功!>>>>>>>>>>>>>");
}
/**
* 获取一个目录下的所有文件
* @param directory ：远程下载目录(以路径符号结束)
* @throws SftpUpDoException
*/
public List<String> listFiles(String directory)
throws SftpUpDoException {
log.info(">>>>>>>>FtpUtil--ftp获取目录文件列表成功!目录:"+directory);
connect();
List<String> allFiles=new ArrayList<>();
try {
sftp.cd(directory);
}
catch (SftpException e) {
try {
sftp.mkdir(directory);
sftp.cd(directory);
}
catch (SftpException e1) {
throw new SftpUpDoException("ftp创建文件路径失败，路径为" + directory);
}
}
try {
Vector files = sftp.ls(".");
Iterator iterator = files.iterator();
while (iterator.hasNext()){
String fileName=((ChannelSftp.LsEntry)iterator.next()).getFilename();
if (".".equals(fileName) || "..".equals(fileName)){
continue;
}
allFiles.add(fileName);
}
}
catch (SftpException e) {
throw new SftpUpDoException("sftp异常-->" + e.getMessage());
}
finally {
disconnect();
log.info(">>>>>>>>FtpUtil--ftp获取目录文件列表成功!目录:"+directory+",文件个数:"+allFiles.size());
return allFiles;
}
}
/**
* 关闭连接
*/
public void disconnect() {
if (this.sftp != null) {
if (this.sftp.isConnected()) {
this.sftp.disconnect();
this.sftp = null;
//log.info("sftp is closed already");
}
}
if (this.sshSession != null) {
if (this.sshSession.isConnected()) {
this.sshSession.disconnect();
this.sshSession = null;
//log.info("sshSession is closed already");
}
}
}
}
ToolUtil.java
public class ToolUtil {
private static final Logger logger = Logger.getLogger("LOGFILE");
private ToolUtil() {
}
public static void doSleep(int second) {
try {
Thread.sleep(second * 1000l);
} catch (Exception localException) {
}
}
/**
* @Description 获取MD5
* @param str
* @return
*/
public static String getMD5(String str) {
MessageDigest digest = null;
BigInteger bigInt = null;
try {
digest = MessageDigest.getInstance("MD5");
digest.update(str.getBytes());
bigInt = new BigInteger(1, digest.digest());
return bigInt.toString(16);
} catch (NoSuchAlgorithmException localNoSuchAlgorithmException) {
}
return "";
}
/**
* @Description 执行shell命令
* @param command
* @return
*/
public static boolean ExecSysCmd(String command) {
logger.info("ExecSysCmd() --" + command);
try {
Runtime rt = Runtime.getRuntime();
Process p = rt.exec(new String[] { "/bin/sh", "-c", command });
return p.waitFor() == 0;
} catch (Exception e) {
logger.error(command, e);
}
return false;
}
/**
* @Description 通过命令移动文件
* @param oldPath
* @param newPath
*/
public static void moveFileCmd(String oldPath, String newPath){
File dirFi=new File(newPath);
if (!dirFi.exists()) {
dirFi.mkdirs();
}
String cmd="mv "+oldPath+" "+newPath;
if (ExecSysCmd(cmd)) {
logger.info("文件:" + oldPath + " 移动到:"+newPath);
}else {
logger.error("文件:" + oldPath + " 移动失败!");
}
}
/**
* @Description 获取一个文件的大小
* @return
*/
public static Object getFileSize(String filePath) {
File file = new File(filePath);
if (!file.exists()) {
logger.info("文件:" + filePath + " 不存在!");
return "0B";
}
long length=file.length();
return getFileSizeByLo(length);
}
/**
* @Description 获取一个文件的大小
* @param length
* @return
*/
public static Object getFileSizeByLo(long length) {
if (length<1024) {
return length+"B";
}else if(length<1024*1024){
return length/1024+"KB";
}else{
return length/(1024*1024) +"MB";
}
}
/**
* @Description 关闭连接
* @param oldPath
* @param ins
* @param ops
*/
public static void closeStram(String oldPath, FileInputStream ins, FileOutputStream ops) {
try {
if (ops!=null) {
ops.close();
}
if (ins!=null) {
ins.close();
}
} catch (IOException e) {
logger.error(oldPath + " 关闭连接异常:" + e.getMessage(), e);
}
}
/**
* @Description 删除文件
* @param filePath
*/
public static void deleteFile(String filePath) {
File file = new File(filePath);
if (!file.exists()) {
logger.info("文件:" + filePath + " 不存在!");
return;
}
if (file.delete()) {
logger.info("文件：" + filePath + " 删除成功!");
}
}
/**
* @Description 判断一个文件是否创建成功
* @param filePath
* @return
*/
public static boolean fileIsCreateSuccess(String filePath){
File file = new File(filePath);
if (!file.exists()) {
logger.info("文件:" + filePath + " 不存在!");
return false;
}
try {
long fron=0l,bac=0l;
bac=file.length();
do {
fron=bac;
Thread.sleep(1000l);
file=new File(filePath);
bac=file.length();
} while (fron<bac);
return true;
} catch (Exception e) {
logger.error("文件创建失败!",e);
return false;
}
}
}
MainConstance.java
public class MainConstance {
private static final Logger logger = Logger.getLogger("LOGFILE");
/** 配置文件路径 **/
public static String CONFIGPATH = "./config.xml";
//public static String CONFIGPATH="./conf/config.xml";
// TODO put操作目录相关
public static List<SftpOpBean> PUT_CONTAINER;
// TODO get操作目录相关
public static List<SftpOpBean> GET_CONTAINER;
/**
* 初始化
* @Description
*/
public static void init() {
logger.info("开始读取配置文件:" + CONFIGPATH);
Document dom = XMLUtils.getDocumentFromClasspath(CONFIGPATH);
/// PUT目录
PUT_CONTAINER = new ArrayList<>();
List<Map<String, String>> incolumns = XMLUtils.getNodeInfoList(dom, "conf/default/put");
if (incolumns.size() > 0) {
for (Map<String, String> column : incolumns) {
String src=column.get("src");
String dst=column.get("dst");
String user=column.get("user");
String pwd=column.get("pwd");
String ip=column.get("ip");
Integer port=Integer.valueOf(column.get("port"));
SftpOpBean bean=new SftpOpBean(src,dst,ip,port,user,pwd);
PUT_CONTAINER.add(bean);
logger.info("PUT_CONTAINER:" + bean.toString());
}
}
/// GET目录
GET_CONTAINER = new ArrayList<>();
List<Map<String, String>> columns = XMLUtils.getNodeInfoList(dom, "conf/default/get");
if (columns.size() > 0) {
for (Map<String, String> column : columns) {
String src=column.get("src");
String dst=column.get("dst");
String user=column.get("user");
String pwd=column.get("pwd");
String ip=column.get("ip");
Integer port=Integer.valueOf(column.get("port"));
SftpOpBean bean=new SftpOpBean(src,dst,ip,port,user,pwd);
GET_CONTAINER.add(bean);
logger.info("GET_CONTAINER:" + bean.toString());
}
}
logger.info("结束读取配置文件:" + CONFIGPATH);
}
}
这个小工具的主要代码大概如上，封装的工具类可以直接使用（小菜，编码水平有待提高！）。
5389
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
