Java实现读取SFTP服务器指定目录文件

宁漂打工仔

已于 2023-10-08 10:36:20 修改

阅读量1.7k

点赞数 1

文章标签： java 服务器 sftp

于 2023-08-17 10:01:14 首次发布

本文链接：https://blog.csdn.net/SpringHASh/article/details/132334351

版权

SFTP服务器的简介

SFTP（SSH File Transfer Protocol）是一种在安全通道上传输文件的协议，它是基于SSH（Secure Shell）协议的扩展，用于在客户端和服务器之间进行加密的文件传输。

SFTP 服务器的主要作用是提供一个安全的方式来上传、下载和管理文件。以下是一些 SFTP 服务器的主要作用：

安全的文件传输： SFTP 使用加密通道传输数据，确保数据在传输过程中不会被窃听或篡改。这使得 SFTP
成为一种安全的文件传输方式，适用于敏感数据或机密文件的传输。
远程文件管理： SFTP
服务器允许用户通过远程连接访问和管理服务器上的文件和目录。用户可以上传、下载、重命名、删除等操作，就像在本地操作文件一样。
备份和归档：组织可以使用 SFTP 服务器来进行文件的备份和归档。通过定期将文件上传到 SFTP
服务器上，可以确保文件的安全存储和恢复。
远程工作： SFTP 服务器使得远程工作变得更加方便。用户可以在任何地方通过 SFTP
客户端访问和处理服务器上的文件，无需物理接触服务器。
数据共享： SFTP 服务器可以用于在团队成员之间共享文件。成员可以通过 SFTP 进行文件的共享和协作，而无需将文件发送给其他人。
自动化流程： SFTP 服务器可以与自动化脚本或工作流程集成，实现自动上传、下载和处理文件，从而提高工作效率。
批量处理： SFTP 服务器支持批量上传和下载文件，适用于需要同时处理多个文件的场景，如数据导入、导出等。

总之，SFTP 服务器提供了一个安全、高效的方式来进行文件传输和管理，适用于许多不同的应用场景，特别是在需要保护数据安全性的情况下。

pom依赖

        <dependency>
            <groupId>com.jcraft</groupId>
            <artifactId>jsch</artifactId>
            <version>${jsch.version}</version>
        </dependency>

     <jsch.version>0.1.55</jsch.version>

实现代码

import cn.hutool.core.text.CharSequenceUtil;
import com.jcraft.jsch.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.FileCopyUtils;

import java.io.*;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.Properties;

/**
 * @author Jalen
 */
public class SftpHelper {

    private static final Logger log = LoggerFactory.getLogger(SftpHelper.class);

    private ChannelSftp sftp;

    private Session session;
    /**
     * SFTP login name
     */
    private String username;
    /**
     * SFTP login password
     */
    private String password;
    /**
     * Private key
     */
    private String privateKey;
    /**
     * SFTP server ip address
     */
    private String host;
    /**
     * SFTP server port
     */
    private Integer port;

    private String directory;

    private List<String> listedFiles;

    private List<String> fileNames;

    /**
     * Construct sftp object based on password authentication
     */
    public SftpHelper(String username, String password, String host, Integer port) {
        this.username = username;
        this.password = password;
        this.host = host;
        this.port = port;
    }

    /**
     * Construct sftp object based on key authentication
     */
    public SftpHelper(String username, String host, Integer port, String privateKey) {
        this.username = username;
        this.host = host;
        this.port = port;
        this.privateKey = privateKey;
    }

    public SftpHelper() {
    }

    public List<String> getFileNames() {
        return this.fileNames;
    }

    public SftpHelper connectSftp(String filePath, String fileName) {
        SftpHelper sftp = new SftpHelper(username, password, host, port);
        sftp.login().find(filePath, fileName);
        if (!sftp.isExistsFile()) {
            log.info(" importDealerData, file no found ");
            return null;
        }
        return sftp;
    }

    /**
     * login sftp server
     */
    public SftpHelper login() {
        try {
            JSch jsch = new JSch();
            if (privateKey != null) {
                // Set private key
                jsch.addIdentity(privateKey);
            }

            session = jsch.getSession(username, host, Optional.ofNullable(port).orElse(0));

            if (password != null) {
                session.setPassword(password);
            }
            Properties config = new Properties();
            config.put("StrictHostKeyChecking", "no");

            session.setConfig(config);
            session.connect();

            Channel channel = session.openChannel("sftp");
            channel.connect();

            sftp = (ChannelSftp) channel;
        } catch (JSchException e) {
            log.warn("sftp login failed, reason:{}", e.getMessage());
        }
        return this;
    }

    /**
     * log out
     */
    public void logout() {
        if (sftp != null) {
            if (sftp.isConnected()) {
                sftp.disconnect();
            }
        }
        if (session != null) {
            if (session.isConnected()) {
                session.disconnect();
            }
        }
    }

    /**
     * Upload the input stream data to sftp as a file. File full path = basePath + directory
     * 
     * @author Jalen
     * @date 2022/8/29 15:46
     * @param basePath
     *            basePath
     * @param directory
     *            directory
     * @param sftpFileName
     *            sftpFileName
     * @param file
     *            file
     */
    public void upload(String basePath, String directory, String sftpFileName, File file) {
        if (file == null || CharSequenceUtil.isEmpty(sftpFileName) || (CharSequenceUtil.isEmpty(basePath) && CharSequenceUtil.isEmpty(directory))) {
            log.error("upload basePath:{} directory:{} sftpFileName:{} failed, reason:{}", basePath, directory, sftpFileName, "Parameters are empty");
            return;
        }

        try {
            if (CharSequenceUtil.isNotEmpty(basePath)) {
                sftp.cd(basePath);
            }
            if (CharSequenceUtil.isNotEmpty(directory)) {
                sftp.cd(directory);
            }
        } catch (SftpException e) {
            // If the directory does not exist, create a folder
            String[] dirs = directory.split("/");
            String tempPath = basePath;
            for (String dir : dirs) {
                if (null == dir || "".equals(dir)) {
                    continue;
                }
                tempPath += "/" + dir;
                try {
                    sftp.cd(tempPath);
                } catch (SftpException ex) {
                    try {
                        sftp.mkdir(tempPath);
                        sftp.cd(tempPath);
                    } catch (SftpException sftpException) {
                        log.error("upload basePath:{} directory:{} sftpFileName:{} failed, reason:{}", basePath, directory, sftpFileName, e.getMessage());
                    }

                }
            }
        }

        InputStream input = null;
        try {
            input = new FileInputStream(file);
            sftp.put(input, sftpFileName);
        } catch (FileNotFoundException | SftpException e) {
            log.error("upload basePath:{} directory:{} sftpFileName:{} failed, reason:{}", basePath, directory, sftpFileName, e.getMessage());
        } finally {
            if (input != null) {
                try {
                    input.close();
                } catch (IOException e) {
                    log.error("upload basePath:{} directory:{} sftpFileName:{} failed, Reason:{}", basePath, directory, sftpFileName, e.getMessage());
                }
            }
        }
    }

    /**
     * download file.
     * 
     * @author Jalen
     * @date 2022/8/29 15:46
     * @param directory
     *            directory
     * @param downloadFile
     *            downloadFile
     * @param saveFile
     *            saveFile
     * @return boolean
     */
    public boolean download(String directory, String downloadFile, String saveFile) throws SftpException, IOException {
        FileOutputStream outPutFile = null;
        try {
            if (CharSequenceUtil.isNotEmpty(directory)) {
                sftp.cd(directory);
            }
            // The folder cannot be operated, otherwise an error will be
            // reported.
            String filepath = saveFile + downloadFile;
            File file = new File(filepath);
            outPutFile = new FileOutputStream(file.getPath());
            sftp.get(downloadFile, outPutFile);
        } catch (Exception e) {
            log.warn("download directory:{} downloadFile:{} failed, reason:{}", directory, downloadFile, e.getMessage());
            return false;
        } finally {
            if (outPutFile != null) {
                outPutFile.close();
            }
        }
        return true;
    }

    /**
     * download file
     * 
     * @author Jalen
     * @date 2022/8/29 15:47
     * @param directory
     *            directory
     * @param downloadFile
     *            downloadFile
     * @return byte[]
     */
    public byte[] download(String directory, String downloadFile) throws SftpException, IOException {
        if (CharSequenceUtil.isNotEmpty(directory)) {
            sftp.cd(directory);
        }
        InputStream is = sftp.get(downloadFile);
        return FileCopyUtils.copyToByteArray(is);
    }

    public InputStream downloadInputStream(String directory, String downloadFile) throws SftpException {
        if (CharSequenceUtil.isNotEmpty(directory)) {
            sftp.cd(directory);
        }
        return sftp.get(downloadFile);
    }

    /**
     * Delete sftp file
     * 
     * @author Jalen
     * @date 2022/8/29 15:47
     * @param directory
     *            directory
     * @param deleteFile
     *            deleteFile
     * @return boolean
     */
    public boolean delete(String directory, String deleteFile) {
        boolean flag;
        try {
            // The full path can be the directory contained in the current
            // directory
            sftp.cd(directory);
            // Delete the full suffixed file name of the file
            sftp.rm(deleteFile);
            flag = true;
        } catch (Exception e) {
            log.info("delete directory:{} deleteFile:{} failed, reason:{}", directory, deleteFile, e.getMessage());
            flag = false;
        }
        return flag;
    }

    public Boolean isValidFile(String fileName) {
        boolean isValid = false;
        if (CharSequenceUtil.isNotEmpty(fileName)) {
            String fileType = fileName.substring(fileName.lastIndexOf(".") + 1);
            boolean exist = CharSequenceUtil.isNotEmpty(fileType) && (fileType.equalsIgnoreCase(Cons.File.CSV)
                    || fileType.equalsIgnoreCase(Cons.File.XLS)
                    || fileType.equalsIgnoreCase(Cons.File.TXT)
                    || fileType.equalsIgnoreCase(Cons.File.XLSX));
            if (exist) {
                isValid = true;
            }
        }
        return isValid;
    }

    public SftpHelper find(String directory, String fileName) {
        this.directory = directory;
        this.listedFiles = listFiles(directory, fileName);
        return this;
    }

    public boolean isExistsFile() {
        return this.listedFiles != null && this.listedFiles.size() > 0;
    }

    private boolean matchFile(String target, String pattern) {
        return target.equals(pattern) || target.matches(pattern);
    }

    public List<String> listFiles(String directory, String fileName) {
        List<String> findFileList = new ArrayList<>();
        ChannelSftp.LsEntrySelector selector = lsEntry -> {
            String lsFileName = lsEntry.getFilename();
            if (CharSequenceUtil.isNotEmpty(fileName)) {
                if (matchFile(lsFileName, fileName)) {
                    findFileList.add(lsFileName);
                }
            } else {
                Boolean isValid = isValidFile(lsFileName);
                if (isValid) {
                    findFileList.add(lsFileName);
                }
            }
            return 0;
        };

        try {
            if (CharSequenceUtil.isNotEmpty(directory)) {
                sftp.ls(directory, selector);
            }
        } catch (SftpException e) {
            log.info("listFiles failed, reason:{}", e.getMessage());
        }

        return findFileList;
    }

    public String getDirectory() {
        return directory;
    }

    public void setDirectory(String directory) {
        this.directory = directory;
    }

    public List<String> getListedFiles() {
        return listedFiles;
    }

    public void setListedFiles(List<String> listedFiles) {
        this.listedFiles = listedFiles;
    }

使用案例

@RequiredArgsConstructor
@Slf4j
@Service
public class SiebelSynFileFacade {

    @Value("${spring.jpa.properties.hibernate.jdbc.batch_size:10}")
    private Integer batchSize;
    @Value("${sftp.test.host}")
    private String host;
    @Value("${sftp.test.port}")
    private Integer port;
    @Value("${sftp.test.username}")
    private String username;
    @Value("${sftp.test.password}")
    private String password;
    @Value("${sftp.test.siebelSynFile.filePath}")
    private String siebelSynFilePath;

    final AccountService accountService;
    final AccountSettingsService accountSettingsService;
    final ServiceReminderService serviceReminderService;
    final BatchService batchService;
    final AccountAreaInterestsService accountAreaInterestsService;
    final BoatService boatService;
    final EngineService engineService;


    public void synData() throws Exception {
        // Only us will synchronize data with siebel
        SftpHelper sftp = this.connectSftp(siebelSynFilePath, null);
        if (ObjectUtil.isNull(sftp)) {
            log.warn(" Files is empty. ");
            return;
        }
        List<String> listedFiles = sftp.getListedFiles();
        // CN 真实场景, 读取ftp目录
        List<BoatSynData> boatEntityList = new ArrayList<>();
        List<EngineSynData> engineEntityList = new ArrayList<>();
        List<AccountEntity> accountEntityList = new ArrayList<>();
        List<AccountSettingsEntity> accountSettingsEntityList = new ArrayList<>();
        List<AccountAreaInterestsEntity> accountAreaInterestsEntityList = new ArrayList<>();
        List<ServiceReminderEntity> serviceReminderEntityList = new ArrayList<>();
        List<ClaimEntity> claimEntityList = new ArrayList<>();
        List<CampaignEntity> campaignEntityList = new ArrayList<>();
        // CN 只读取当天文件
        for (String listedFile : listedFiles) {
            String name = listedFile;
            InputStream fileInputStream = sftp.downloadInputStream(siebelSynFilePath, listedFile);
            if (CharSequenceUtil.isBlank(name)) {
                throw new IllegalArgumentException(" Filename is not empty.");
            }
            name = name.substring(0, name.indexOf(Cons.Delimiter.UNDERSCORE));
            switch (name) {
                case SiebelCons.BOAT ->
                        boatEntityList = SynFileProxy.parseFile(name, fileInputStream, BoatSynData.class);
                case SiebelCons.ENGINE ->
                        engineEntityList = SynFileProxy.parseFile(name, fileInputStream, EngineSynData.class);
                case SiebelCons.ACCOUNT ->
                        accountEntityList = SynFileProxy.parseFile(name, fileInputStream, AccountEntity.class);
                case SiebelCons.ACCOUNT_SETTINGS ->
                        accountSettingsEntityList = SynFileProxy.parseFile(name, fileInputStream, AccountSettingsEntity.class);
                case SiebelCons.SERVICE_REMINDER ->
                        serviceReminderEntityList = SynFileProxy.parseFile(name, fileInputStream, ServiceReminderEntity.class);
                case SiebelCons.CLAIM ->
                        claimEntityList = SynFileProxy.parseFile(name, fileInputStream, ClaimEntity.class);
                case SiebelCons.AREA_INT ->
                        accountAreaInterestsEntityList = SynFileProxy.parseFile(name, fileInputStream, AccountAreaInterestsEntity.class);
                case SiebelCons.CAMPAIGN ->
                        campaignEntityList = SynFileProxy.parseFile(name, fileInputStream, CampaignEntity.class);
                default -> {
                }
            }
        }
        // CN 合并数据入库
        this.handleAccount(accountEntityList);
        this.handleAccountSettings(accountSettingsEntityList);
        this.handleReminder(serviceReminderEntityList);
        this.handleClaim(claimEntityList);
        this.handleBoat(boatEntityList);
        this.handleEngine(engineEntityList);
        this.handleAccountAreaInterests(accountAreaInterestsEntityList);
        this.handleCampaign(campaignEntityList);
        // CN 操作完成删除文件
        for (String listedFile : listedFiles) {
            sftp.delete(siebelSynFilePath, listedFile);
        }
    }


    /*------------------------------ private ------------------------------*/

    private SftpHelper connectSftp(String filePath, String fileName) {
        SftpHelper sftp = new SftpHelper(username, password, host, port);
        sftp.login().find(filePath, fileName);
        if (!sftp.isExistsFile()) {
            log.info(" importDealerData, file no found ");
            return null;
        }
        return sftp;
    }