从sftp到oss再到数据库

1.SftpTools工具类

package com.util;

import com.jcraft.jsch.*;
import lombok.extern.slf4j.Slf4j;
import java.io.InputStream;
import java.util.*;

@Slf4j
public class SftpTools {

	private  static Session sshSession;
	/**
	 * 连接sftp服务器
	 * @param host：ftp地址；port：端口；userName：账号；password：密码
	 */
	public static ChannelSftp sftpConnection(String host,int port,String userName,String password){
		JSch jsch = new JSch();
		ChannelSftp channelSftp = null;
		try {
			jsch.getSession(userName,host,port);
			sshSession = jsch.getSession(userName,host,port);
			sshSession.setPassword(password);
			Properties properties = new Properties();
			properties.put("StrictHostKeyChecking", "no");
			sshSession.setConfig(properties);
			sshSession.connect();
			Channel channel = sshSession.openChannel("sftp");
			channel.connect();
			channelSftp = (ChannelSftp) channel;
		} catch (JSchException e) {
			log.error("连接SFTP服务器异常：{}",e.getMessage());
		}
		return  channelSftp;
	}

	/**
	 *@description 退出Sftp服务器登录
	 *@return
	 **/
	public static void sftpClose(ChannelSftp channelSftp){
		if (channelSftp != null) {
			if (channelSftp.isConnected()){
				channelSftp.disconnect();
			}
		}
	}

	/**
	 * 关闭session
	 */
	public static void sessionClose(){
		if (sshSession != null) {
			if (sshSession.isConnected()){
				sshSession.disconnect();
				sshSession = null;
			}
		}
	}

	/**
	 * 下载sftp路径下所有 文件
	 * @param sftp
	 * @param path 文件路径
	 * @param date 日期 eg:20230321
	 * @throws Exception
	 * @return
	 */
	public static Map<String,Map<String, InputStream>> downSftpFile(ChannelSftp sftp, String path, String date){
		Map<String,Map<String, InputStream>> mapPdate=new HashMap<>();
		Map<String,InputStream> map = new HashMap<>();
		try {
			if (path != null && !"".equals(path)) {
				sftp.cd(path);//进入所在路径
			}


			Vector vector = sftp.ls("./");
			Iterator iterator = vector.iterator();
			while (iterator.hasNext()) {
				ChannelSftp.LsEntry file = (ChannelSftp.LsEntry) iterator.next();
				//文件名称
				String fileName = file.getFilename();
				if (!fileName.startsWith(".")) {
					if (fileName.contains(date)) {
						InputStream inputStream = sftp.get(fileName);
						if (inputStream != null) {
							map.put(fileName, inputStream);
						}
					}
				}
			}
			mapPdate.put(date,map);
		} catch (SftpException e) {
			e.printStackTrace();
		}
		return mapPdate;
	}


	/**
	 * 下载sftp中符合条件的所有文件
	 * @param sftp
	 * @param path 文件路径
	 * @param date 日期 eg:20230321
	 * @throws Exception
	 * @return
	 */
	public static Map<String,Map<String, InputStream>> autoDownSftpFile(ChannelSftp channelSftp, String path, List<String> pdates, List<String> fileNameList) {
		Map<String,Map<String, InputStream>> mapPdate=new HashMap<>();
		Map<String,InputStream> map;
		try {
			if (path != null && !"".equals(path)) {
				channelSftp.cd(path);//进入所在路径
			}
			Vector vector = channelSftp.ls("./");
			for (Object obj : vector) {
				ChannelSftp.LsEntry file = (ChannelSftp.LsEntry) obj;
				//文件名称
				String fileName = file.getFilename();
				for (String name : fileNameList) {
					if (!fileName.startsWith(".")) {
						if (fileName.contains(name)) {
							for (String pdate : pdates) {
								if (fileName.contains(pdate)) {
									InputStream inputStream = channelSftp.get(fileName);
									if (inputStream != null) {
										if(mapPdate.get(pdate)==null){
											map = new HashMap<>();
										}else{
											map = mapPdate.get(pdate);
										}
										map.put(fileName, inputStream);
										mapPdate.put(pdate,map);
									}
								}

							}
						}
					}
				}
			}
		} catch (SftpException e) {
			log.error("自动获取SFTP文件异常：{}",e.getMessage());
		}
		return mapPdate;
	}
}

2.OssTools工具类

package com.util;

import com.aliyun.oss.*;
import com.aliyun.oss.model.OSSObject;
import com.aliyun.oss.model.PutObjectRequest;
import com.aliyun.oss.model.PutObjectResult;
import lombok.extern.slf4j.Slf4j;
import java.io.*;
import java.util.*;

@Slf4j
public class OssTools {

	//	从OSS读取文件流
	public static Map<String,List<String>> getBufferedReader(String endpoint, String accessKeyId, String accessKeySecret, String bucketName, String name) {
		// 创建OSSClient实例。
		ClientBuilderConfiguration conf = new ClientBuilderConfiguration();
		// 私有云要关闭CNAME
		conf.setSupportCname(false);
		OSS ossClient = new OSSClientBuilder().build(endpoint, accessKeyId, accessKeySecret,conf);

		OSSObject ossObject = null;
		BufferedReader reader = null;
		List<String> list = new ArrayList<>();
		Map<String,List<String>> map=new HashMap<>();
		try {
			// ossObject包含文件所在的存储空间名称、文件名称、文件元信息以及一个输入流。
			ossObject = ossClient.getObject(bucketName, name);

			// 读取文件内容。
			reader = new BufferedReader(new InputStreamReader(ossObject.getObjectContent(),"utf-8"));
			while (true) {
				String line = reader.readLine();
				if (line == null) break;
				else list.add(line);
			}
		} catch (Exception e) {
			log.error("导入失败");
		}finally {
			try {
				// 数据读取完成后，获取的流、对象必须关闭，否则会造成连接泄漏，导致请求无连接可用，程序无法正常工作。
				if (reader != null ) reader.close();
				if (ossObject != null ) ossObject.close();
			} catch (IOException e) {
				log.error("数据读取流或ossObject对象使用完毕后未成功关闭");
			}
			if (ossClient != null) ossClient.shutdown();
		}
		return map;
	}

	//	往oss上传文件；date 形式 20230321
	public static boolean uploadfile(String endpoint, String accessKeyId, String accessKeySecret, String bucketName, String path,List<String> dates , Map<String, Map<String, InputStream>> inputStreamMapPdate) {

		// 创建OSSClient实例。
		ClientBuilderConfiguration conf = new ClientBuilderConfiguration();
		// 私有云要关闭CNAME
		conf.setSupportCname(false);
		OSS ossClient = new OSSClientBuilder().build(endpoint, accessKeyId, accessKeySecret,conf);
		//自己电脑测试用
//				OSS ossClient = new OSSClientBuilder().build(endpoint, accessKeyId, accessKeySecret);
		boolean flag=false;
		try {
			for (String date:dates){
				Map<String, InputStream> inputStreamMap = inputStreamMapPdate.get(date);
				if (inputStreamMap != null){
					String name = path+ date + "/";
				//			判断指定日期的文件夹是否存在
					boolean found = ossClient.doesObjectExist(bucketName, name);
					if (!found){//先创建文件夹

						String content = "";
						// 创建PutObjectRequest对象。
						PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, name, new ByteArrayInputStream(content.getBytes()));
						// 上传字符串。
						ossClient.putObject(putObjectRequest);
					}
					Set<String> strings = inputStreamMap.keySet();
					for (String keyfile : strings) {
						// 创建PutObjectRequest对象。
						PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, name+keyfile, inputStreamMap.get(keyfile));
						// 设置该属性可以返回response。如果不设置，则返回的response为空。
						putObjectRequest.setProcess("true");
						// 创建PutObject请求。
						PutObjectResult result = ossClient.putObject(putObjectRequest);
					}
				}
			}
			log.info("上传成功");
			flag = true;
		} catch (Exception e) {
			log.info("上传失败");
			log.error(String.valueOf(e));
		}finally {
			if (null!=ossClient){
				ossClient.shutdown();
			}
			return flag;
		}

	}

}

3.java调用

1）从sftp获取文件至oss

	@GetMapping("/sftpToOss")
	public R sftpToOss() {
//		获取需要从sftp读取的文件列表（后缀包含日期和文件名）
		List<String> fileNameList = new ArrayList<>();
		fileNameList.add("文件一");
		fileNameList.add("文件二");
		List<String> dates = new ArrayList<>();
		dates.add("20240415");
		dates.add("20240416");
		
//		建立sftp连接
		ChannelSftp channelSftp = null;
		try {
			channelSftp = SftpTools.sftpConnection(host, port, username, password);
		} catch (JSchException e) {
			log.error("连接SFTP服务器异常：{}",e.getMessage());
			return R.ok("连接SFTP服务器异常!!");
		}

//		获取sftp文件流
		Map<String, Map<String, InputStream>> inputStreamMap = SftpTools.autoDownSftpFile(channelSftp, path, dates, fileNameList);

//		建立oss连接--保存文件流--关闭oss连接
		if (!inputStreamMap.isEmpty()) {
			OssTools.uploadfile(endpoint, accessKeyId, accessKeySecret, bucketName, interfaceDdzx, dates, inputStreamMap);
		}
		SftpTools.sessionClose();
		SftpTools.sftpClose(channelSftp);
		
		return R.ok(true);
	}

2）从oss获取文件至数据库

	public String readRB(String pdate,String username) {
//		获取oss中文件流
		Map<String, List<String>> bufferedReader = OssTools.getBufferedReader(endpoint, accessKeyId, accessKeySecret, bucketName, "文件.RB");
		Set<String> strings = bufferedReader.keySet();
		Iterator<String> iterator = strings.iterator();
		String typek = iterator.next();
		boolean flag = true;

		if (typek.equals("2")) {
			return bufferedReader.get(typek).get(0);
		} else {
			List<String> fileContent = bufferedReader.get(typek);
			List<DataAheadTrade> list = new ArrayList<>();
			if (ObjectUtils.isNotEmpty(fileContent)) {
				//导入RB文件全部内容
				for (String info : fileContent) {
					if (info.startsWith("#")) {
						String[] infos = info.split("\\t+");
						Data entity = new Data();
						entity.setPeriodid(Convert.toInt(infos[2]));
						entity.setPrice(Convert.toBigDecimal(infos[3]));
						list.add(entity);
					}
				}

				if (ObjectUtils.isNotEmpty(list)) {
					this.remove(new QueryWrapper<Data>().lambda().eq(Data::getPeriodid,periodid));
					flag = this.saveBatch(list);
				}
			}
			if (flag) {
				return "导入RB成功";
			} else {
				return "导入文件失败";
			}
		}
	}