开心一刻
今天逛街碰到街头采访,一上来就问我敏感话题
主持人:小哥哥,你单身吗
我:是啊
主持人:你找女朋友的话,是想找一个小奶猫呢,还是小野猫呢
我沉思了一下,叹气道:如果可以的话,我想找个人,而且是女人
上传文件
基于 commons-vfs2
实现文件到 FTP
服务器的上传,pom.xml 如下
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.qsl</groupId>
<artifactId>ftp-demo</artifactId>
<version>1.0-SNAPSHOT</version>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.7.18</version>
</parent>
<properties>
<maven.compiler.source>8</maven.compiler.source>
<maven.compiler.target>8</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-vfs2</artifactId>
<version>2.7.0</version>
</dependency>
<dependency>
<groupId>commons-net</groupId>
<artifactId>commons-net</artifactId>
<version>3.9.0</version>
</dependency>
</dependencies>
</project>
- 1.
- 2.
- 3.
- 4.
- 5.
- 6.
- 7.
- 8.
- 9.
- 10.
- 11.
- 12.
- 13.
- 14.
- 15.
- 16.
- 17.
- 18.
- 19.
- 20.
- 21.
- 22.
- 23.
- 24.
- 25.
- 26.
- 27.
- 28.
- 29.
- 30.
- 31.
- 32.
- 33.
- 34.
- 35.
- 36.
- 37.
- 38.
- 39.
application.yml
FtpConfig.java
/**
* FTP 配置
* @author 青石路
*/
@Configuration
@ConfigurationProperties(prefix = "app.ftp")
public class FtpConfig {
private String host;
private String userName;
private String password;
private Integer port;
private String protocol;
private String baseDir;
@Bean("fptUri")
public URI fptUri() throws URISyntaxException {
return new URI(protocol, userName+":"+password, host,
port, baseDir, null, null);
}
@Bean
public FileSystemOptions fileSystemOptions() {
FileSystemOptions opts = new FileSystemOptions();
FtpFileSystemConfigBuilder builder = FtpFileSystemConfigBuilder.getInstance();
builder.setControlEncoding(opts, "UTF-8");
builder.setConnectTimeout(opts, 5000);
builder.setUserDirIsRoot(opts, true);
builder.setPassiveMode(opts, true);
return opts;
}
public String getHost() {
return host;
}
public void setHost(String host) {
this.host = host;
}
public String getUserName() {
return userName;
}
public void setUserName(String userName) {
this.userName = userName;
}
public String getPassword() {
return password;
}
public void setPassword(String password) {
this.password = password;
}
public Integer getPort() {
return port;
}
public void setPort(Integer port) {
this.port = port;
}
public String getProtocol() {
return protocol;
}
public void setProtocol(String protocol) {
this.protocol = protocol;
}
public String getBaseDir() {
return baseDir;
}
public void setBaseDir(String baseDir) {
this.baseDir = baseDir;
}
}
- 1.
- 2.
- 3.
- 4.
- 5.
- 6.
- 7.
- 8.
- 9.
- 10.
- 11.
- 12.
- 13.
- 14.
- 15.
- 16.
- 17.
- 18.
- 19.
- 20.
- 21.
- 22.
- 23.
- 24.
- 25.
- 26.
- 27.
- 28.
- 29.
- 30.
- 31.
- 32.
- 33.
- 34.
- 35.
- 36.
- 37.
- 38.
- 39.
- 40.
- 41.
- 42.
- 43.
- 44.
- 45.
- 46.
- 47.
- 48.
- 49.
- 50.
- 51.
- 52.
- 53.
- 54.
- 55.
- 56.
- 57.
- 58.
- 59.
- 60.
- 61.
- 62.
- 63.
- 64.
- 65.
- 66.
- 67.
- 68.
- 69.
- 70.
- 71.
- 72.
- 73.
- 74.
- 75.
- 76.
- 77.
- 78.
- 79.
- 80.
FileUploadManager.java 完成上传
/**
* 文件上传 manager
* @author 青石路
*/
@Component
public class FileUploadManager {
private static final Logger LOGGER = LoggerFactory.getLogger(FileUploadManager.class);
private static final FileSystemManager systemManager;
@Autowired
@Qualifier("fptUri")
private URI fptUri;
@Autowired
private FileSystemOptions fileSystemOptions;
static {
try {
systemManager = VFS.getManager();
} catch (FileSystemException e) {
throw new RuntimeException(e);
}
}
public boolean uploadFileToSftp(File file, String fileName) {
try {
FileObject srcObject = systemManager.resolveFile(file.getParentFile(), file.getName());
FileObject destObjectDir = systemManager.resolveFile(fptUri.toString(), fileSystemOptions);
FileObject destObject = systemManager.resolveFile(destObjectDir, fileName);
destObject.copyFrom(srcObject, Selectors.SELECT_SELF);
return true;
} catch (FileSystemException e) {
LOGGER.error("文件:{} 上传SFTP失败,异常:", file.getAbsoluteFile(), e);
return false;
}
}
}
- 1.
- 2.
- 3.
- 4.
- 5.
- 6.
- 7.
- 8.
- 9.
- 10.
- 11.
- 12.
- 13.
- 14.
- 15.
- 16.
- 17.
- 18.
- 19.
- 20.
- 21.
- 22.
- 23.
- 24.
- 25.
- 26.
- 27.
- 28.
- 29.
- 30.
- 31.
- 32.
- 33.
- 34.
- 35.
- 36.
- 37.
FileUploadController.java
/**
* 文件上传 controller
* @author 青石路
*/
@RestController
@RequestMapping("file")
public class FileUploadController {
private static final Logger LOGGER = LoggerFactory.getLogger(FileUploadController.class);
@Autowired
private FileUploadManager fileUploadManager;
@GetMapping("upload")
public boolean upload(@RequestParam("fileName") String fileName) {
long ramdomLong = ThreadLocalRandom.current().nextLong();
File file = new File(System.getProperty("java.io.tmpdir") + File.separator + ramdomLong + ".txt");
LOGGER.info("localFile:{}", file.getAbsoluteFile());
boolean uploadResult = false;
try {
Files.write(file.toPath(), String.valueOf(ramdomLong).getBytes(StandardCharsets.UTF_8));
uploadResult = fileUploadManager.uploadFileToSftp(file, fileName);
} catch (IOException e) {
throw new RuntimeException(e);
} finally {
try {
Files.deleteIfExists(file.toPath());
} catch (IOException e) {
throw new RuntimeException(e);
}
}
return uploadResult;
}
}
- 1.
- 2.
- 3.
- 4.
- 5.
- 6.
- 7.
- 8.
- 9.
- 10.
- 11.
- 12.
- 13.
- 14.
- 15.
- 16.
- 17.
- 18.
- 19.
- 20.
- 21.
- 22.
- 23.
- 24.
- 25.
- 26.
- 27.
- 28.
- 29.
- 30.
- 31.
- 32.
- 33.
- 34.
完整代码: ftp-demo
FTP 服务器目录最初情况如下
/idg
下没有任何目录和文件;启动后调用接口
http://localhost:8080/file/upload?fileName=hello.txt
即可完成文件的上传;fileName
参数表示上传到 FTP 服务器上的文件名
true
表示上传成功,FTP 服务器上即可看到 hello.txt
file
目录也被自动创建了,一切都是那么的顺利
上传失败:553
一个不小心把 FTP 服务器上 file
目录给删了,但内心一点都不慌,再上传一次呗,正好我也是这么干的;正当我以为会正常上传的时候,意外来了
org.apache.commons.vfs2.FileSystemException: Could not copy "file:///C:/Users/qsl/AppData/Local/Temp/6456333409667879871.txt" to "ftp://test:***@192.168.2.118/idg/file/hello.txt".
at org.apache.commons.vfs2.provider.AbstractFileObject.copyFrom(AbstractFileObject.java:303)
at com.qsl.manager.FileUploadManager.uploadFileToSftp(FileUploadManager.java:47)
at com.qsl.web.FileUploadController.upload(FileUploadController.java:39)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
at org.springframework.web.servlet.FrameworkServlet.doGet(FrameworkServlet.java:898)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:529)
at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:623)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:209)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:153)
at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:51)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:178)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:153)
at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:178)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:153)
at org.springframework.web.filter.FormContentFilter.doFilterInternal(FormContentFilter.java:93)
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:178)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:153)
at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:178)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:153)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:168)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:90)
at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:481)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:130)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:93)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:74)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:342)
at org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:390)
at org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:63)
at org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:928)
at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1794)
at org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:52)
at org.apache.tomcat.util.threads.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1191)
at org.apache.tomcat.util.threads.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:659)
at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.commons.vfs2.FileSystemException: Could not write to "ftp://test:***@192.168.2.118/idg/file/hello.txt".
at org.apache.commons.vfs2.provider.AbstractFileObject.getOutputStream(AbstractFileObject.java:1280)
at org.apache.commons.vfs2.provider.DefaultFileContent.buildOutputStream(DefaultFileContent.java:540)
at org.apache.commons.vfs2.provider.DefaultFileContent.getOutputStream(DefaultFileContent.java:406)
at org.apache.commons.vfs2.provider.DefaultFileContent.getOutputStream(DefaultFileContent.java:394)
at org.apache.commons.vfs2.provider.DefaultFileContent.write(DefaultFileContent.java:815)
at org.apache.commons.vfs2.provider.DefaultFileContent.write(DefaultFileContent.java:830)
at org.apache.commons.vfs2.util.FileObjectUtils.writeContent(FileObjectUtils.java:203)
at org.apache.commons.vfs2.provider.AbstractFileObject.copyFrom(AbstractFileObject.java:298)
... 52 common frames omitted
Caused by: org.apache.commons.vfs2.FileSystemException: Cant open output connection for file "ftp://test:***@192.168.2.118/idg/file/hello.txt". Reason: "553 Can't open that file: No such file or directory
".
at org.apache.commons.vfs2.FileSystemException.requireNonNull(FileSystemException.java:87)
at org.apache.commons.vfs2.provider.ftp.FtpFileObject.doGetOutputStream(FtpFileObject.java:519)
at org.apache.commons.vfs2.provider.AbstractFileObject.getOutputStream(AbstractFileObject.java:1276)
... 59 common frames omitted
- 1.
- 2.
- 3.
- 4.
- 5.
- 6.
- 7.
- 8.
- 9.
- 10.
- 11.
- 12.
- 13.
- 14.
- 15.
- 16.
- 17.
- 18.
- 19.
- 20.
- 21.
- 22.
- 23.
- 24.
- 25.
- 26.
- 27.
- 28.
- 29.
- 30.
- 31.
- 32.
- 33.
- 34.
- 35.
- 36.
- 37.
- 38.
- 39.
- 40.
- 41.
- 42.
- 43.
- 44.
- 45.
- 46.
- 47.
- 48.
- 49.
- 50.
- 51.
- 52.
- 53.
- 54.
- 55.
- 56.
- 57.
- 58.
- 59.
- 60.
- 61.
- 62.
- 63.
- 64.
- 65.
- 66.
- 67.
- 68.
- 69.
- 70.
553 Can't open that file: No such file or directory 是什么鬼?莫非是 file
目录不存在的原因?试着手动创建 file
目录,再调用接口上传文件,文件正常上传!
我们来分析下,最初的时候 file
目录是不存在的,但自动创建了,文件也正常上传了,然后我们手动删除 file
目录后,上传文件失败,手动补上 file
目录后,上传又正常了,这说明 file
目录被缓存了呀,对不对?最初的时候,缓存是空的,第一次上传的时候,vfs2
会判断 FTP 服务器上是否存在 file
目录,不存在则创建并进行缓存,那么下次上传的时候,在缓存中找到了 file
目录,那么就直接上传文件了,而不用去判断 FTP 服务器上是否有 file
目录(没有则创建);缓存的作用就很明显了,减少了一次目录是否存在的网络请求,进而提高效率;当然这只是我们的猜想,是否真的存在缓存,看源码肯定是最直观的,入口代码
FileObject destObjectDir = systemManager.resolveFile(fptUri.toString(), fileSystemOptions);
就不带你们详细去跟源码了,debug
跟源码,我相信你们也很容易找到 AbstractFileSystem#resolveFile(final FileName name, final boolean useCache)
private synchronized FileObject resolveFile(final FileName name, final boolean useCache)
throws FileSystemException {
if (!rootName.getRootURI().equals(name.getRootURI())) {
throw new FileSystemException("vfs.provider/mismatched-fs-for-name.error", name, rootName,
name.getRootURI());
}
// imario@apache.org ==> use getFileFromCache
FileObject file;
if (useCache) {
file = getFileFromCache(name);
} else {
file = null;
}
if (file == null) {
try {
file = createFile((AbstractFileName) name);
} catch (final Exception e) {
throw new FileSystemException("vfs.provider/resolve-file.error", name, e);
}
file = decorateFileObject(file);
// imario@apache.org ==> use putFileToCache
if (useCache) {
putFileToCache(file);
}
}
/**
* resync the file information if requested
*/
if (getFileSystemManager().getCacheStrategy().equals(CacheStrategy.ON_RESOLVE)) {
file.refresh();
}
return file;
}
- 1.
- 2.
- 3.
- 4.
- 5.
- 6.
- 7.
- 8.
- 9.
- 10.
- 11.
- 12.
- 13.
- 14.
- 15.
- 16.
- 17.
- 18.
- 19.
- 20.
- 21.
- 22.
- 23.
- 24.
- 25.
- 26.
- 27.
- 28.
- 29.
- 30.
- 31.
- 32.
- 33.
- 34.
- 35.
- 36.
- 37.
- 38.
如何修复
首先我们讨论下:要不要修?
手动误删目录,这种情况是非常少的,就拿我们的生产来讲,2020 到现在,从未出现过该问题,如果因为这种极小概率的事件去放弃缓存带来的性能提升,得不偿失,所以我是不推荐修改的,而实际上经过讨论后也决定不去修改;不过话说回来,如果上传压力很小,网络又非常稳定、快速,也就说缓存带来的性能提升可以忽略,那这个时候高可用的优先级就更高了,此时就可以考虑修了,那怎么修了,我这里提供两种方案
- 禁用缓存
既然vfs2
有缓存(默认是启用的),应该有开关来禁用它,我就不给你们打哑谜了,直接修改FileSystemManager
,换成其子类StandardFileSystemManager
private static final StandardFileSystemManager systemManager;
static {
try {
systemManager = new StandardFileSystemManager();
// 禁用缓存
systemManager.setFilesCache(new NullFilesCache());
systemManager.init();
} catch (FileSystemException e) {
throw new RuntimeException(e);
}
}
- 1.
- 2.
- 3.
- 4.
- 5.
- 6.
- 7.
- 8.
- 9.
- 10.
- 11.
- 12.
你们也许会问,我是怎么知道可以这么调整的,你们跟一下 VFS.getManager()
就知道了,最终会看到
// managerClassName: org.apache.commons.vfs2.impl.StandardFileSystemManager
private static FileSystemManager createFileSystemManager(final String managerClassName) throws FileSystemException {
try {
// Create instance
final Class<?> mgrClass = Class.forName(managerClassName);
final FileSystemManager mgr = (FileSystemManager) mgrClass.newInstance();
try {
// Initialize
final Method initMethod = mgrClass.getMethod("init", (Class[]) null);
initMethod.invoke(mgr, (Object[]) null);
} catch (final NoSuchMethodException ignored) {
/* Ignore; don't initialize. */
}
return mgr;
} catch (final InvocationTargetException e) {
throw new FileSystemException("vfs/create-manager.error", managerClassName, e.getTargetException());
} catch (final Exception e) {
throw new FileSystemException("vfs/create-manager.error", managerClassName, e);
}
}
- 1.
- 2.
- 3.
- 4.
- 5.
- 6.
- 7.
- 8.
- 9.
- 10.
- 11.
- 12.
- 13.
- 14.
- 15.
- 16.
- 17.
- 18.
- 19.
- 20.
- 21.
- 22.
通过反射调用了 StandardFileSystemManager
的构造方法和 init
方法,与我们的
是不是有异曲同工之妙?(你们猜的没错,我们的实现正是抄自于 vfs2
)
- 异常弥补
不禁用缓存,还是保留默认的开启,只是当异常的时候,捕获它,然后去创建目录,然后再上传一次
public boolean uploadFileToSftp(File file, String fileName, boolean isRetry) {
String destDir = fptUri.toString();
try {
FileObject srcObject = systemManager.resolveFile(file.getParentFile(), file.getName());
FileObject destObjectDir = systemManager.resolveFile(destDir, fileSystemOptions);
FileObject destObject = systemManager.resolveFile(destObjectDir, fileName);
destObject.copyFrom(srcObject, Selectors.SELECT_SELF);
return true;
} catch (FileSystemException e) {
if (isRetry) {
try {
LOGGER.info("创建目录{}开始", destDir);
FileObject destObjectDir = systemManager.resolveFile(destDir, fileSystemOptions);
destObjectDir.createFolder();
LOGGER.info("创建目录{}开始", destDir);
} catch (FileSystemException ex) {
LOGGER.error("创建目录失败,异常:", ex);
}
return uploadFileToSftp(file, fileName, false);
}
LOGGER.error("文件:{} 上传SFTP失败,异常:", file.getAbsoluteFile(), e);
return false;
}
}
- 1.
- 2.
- 3.
- 4.
- 5.
- 6.
- 7.
- 8.
- 9.
- 10.
- 11.
- 12.
- 13.
- 14.
- 15.
- 16.
- 17.
- 18.
- 19.
- 20.
- 21.
- 22.
- 23.
- 24.
既保留了缓存,也解决了目录误删的问题,就问你们服不服?
总结
vfs2
是有缓存的,如果不小心把 FTP 目录删除了,上传会失败并提示
553 Can't open that file: No such file or directory
可以通过手动补目录的方式就行处理,当然也可以通过重启服务来解决,但这两种都不是通过代码来解决的,可用性很低;通过代码的方式来解决,有两种方法
- 禁用
vfs2
缓存,但会降低性能,可用但不推荐 - 异常弥补,既保留了缓存,也解决了目录误删的问题,可以用也推荐
异常捕获后用来做流程控制,条件控制,不太规范