java ftp批量下载文件，偶尔出现空文件问题

 

■1.问题现象

ftp批量下载文件时，偶尔出现空文件。

■2.问题原因

A:代码环境

整体环境:springboot2.0.4

    <parent>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-parent</artifactId>
        <version>2.0.4.RELEASE</version>
        <relativePath/> <!-- lookup parent from repository -->
    </parent>

 ftp环境：org.apache.commons.net.ftp

        <dependency>
            <groupId>commons-net</groupId>
            <artifactId>commons-net</artifactId>
            <version>3.3</version>
        </dependency>

    /**
     * 下载指定目录下指定后缀名文件
     *
     * @param hostname  FTP服务器地址
     * @param port      FTP服务器端口号
     * @param username  FTP登录帐号
     * @param password  FTP登录密码
     * @param pathname  FTP服务器文件目录
     * @param localpath 下载后的文件路径
     * @return
     */
    public static List<String> downloadFileForSubfix(String hostname, int port, String username, String password, String pathname,
                                                     int spanId, String localpath, String[] subfixs, Integer connectId) {
        List<String> fileNames = new ArrayList<>();
        List<FTPFile> fTPFiles = new ArrayList<>();
        // org.apache.commons.net.ftp
        FTPClient ftp = null;
        try {
            ftp = getFTPClient(hostname, username, password, port);
            // 主动模式
            ftp.enterLocalActiveMode();
            // 设置文件类型  二进制文件
            ftp.setFileType(FTPClient.BINARY_FILE_TYPE);
            // 切换目录
            boolean flag = ftp.changeWorkingDirectory(new String(pathname.getBytes(), FTP.DEFAULT_CONTROL_ENCODING));
            if (!flag) {
                throw new Exception("目录[" + pathname + "]不存在");
            }
            FTPFile[] fs = ftp.listFiles(pathname, file -> {
                String name = file.getName();
                String strSubName = name.substring(name.lastIndexOf("."));
                for (String subfix : subfixs) {
                    if (subfix.equalsIgnoreCase(strSubName)) {
                        return true;
                    }
                }
                return false;
            });
            for (FTPFile ftpFile : fs) {
                if(ftpFile.isFile()){
                    fTPFiles.add(ftpFile);
                }
            }
            //文件排序--时间早的文件优先处理
            Collections.sort(fTPFiles, (file, newFile) -> {
                if (file.getTimestamp().getTimeInMillis() < newFile.getTimestamp().getTimeInMillis()) {
                    return -1;
                } else if (file.getTimestamp().getTimeInMillis() == newFile.getTimestamp().getTimeInMillis()) {
                    return 0;
                } else {
                    return 1;
                }
            });
            for (int i = 0; i < fTPFiles.size(); i++) {
                if (i >= DiConstants.FTP_FILE_COUNT_20s) {
                    break;
                }
                FTPFile ff = fTPFiles.get(i);
                String strNewName = writeFile(ff, localpath, ftp, connectId);
                //deleteFile(hostname, port, username, password, pathname, strNewName);
                fileNames.add(strNewName);
            }
        } catch (Exception e) {
            logger.error("FTP下载文件异常{}", e);
        } finally {
            try {
                if (ftp != null) {
                    ftp.logout();
                }
            } catch (IOException e1) {
                e1.printStackTrace();
            }

            if (ftp.isConnected()) {
                try {
                    ftp.disconnect();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
        return fileNames;
    }

    public static String writeFile(FTPFile ff, String localpath, FTPClient ftp, Integer connectId) {
        FileOutputStream out = null;
        InputStream in = null;
        String outFileName = ff.getName();
        String strConnect = "CONNECT";
        String strNewName = strConnect.concat(DiConstants.MARK_LEFT_BRACKET).concat(String.format("%03d", connectId))
                .concat(DiConstants.MARK_RIGHT_BRACKET).concat(outFileName);
        try {
            File localFile = new File(localpath.concat(File.separator).concat(strNewName));
            out = new FileOutputStream(localFile);
            //TODO 默认为ASCII文件编码格式但是需要改成二进制
            ftp.setFileType(FTP.BINARY_FILE_TYPE);
            in = ftp.retrieveFileStream(outFileName);
            byte[] byteArray = new byte[4096];
            int read = 0;
            while ((read = in.read(byteArray)) != -1) {
                out.write(byteArray, 0, read);
                out.flush();
            }
            // 要多次操作这个ftp的流的通道,要等他的每次命令完成
            ftp.completePendingCommand();   //TODO 这个有点问题
        } catch (FileNotFoundException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            //TODO 为什么要这么写？？？？
            if (null != out) {
                try {
                    out.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
            if (null != in) {
                try {
                    in.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
        return strNewName;

    }

经测试发现[in = ftp.retrieveFileStream(outFileName);]中获取的in中未获取读出内容(猜测可能是读取文件偶尔会很慢，造成了阻塞)，程序无法进入后面的whil循环，导致空文件。 

■3.解决办法

    /**
     * 下载指定目录下指定后缀名文件
     *
     * @param hostname  FTP服务器地址
     * @param port      FTP服务器端口号
     * @param username  FTP登录帐号
     * @param password  FTP登录密码
     * @param pathname  FTP服务器文件目录
     * @param localpath 下载后的文件路径
     * @return
     */
    public static List<String> downloadFileForSubfix(String hostname, int port, String username, String password, String pathname,
	            int spanId, String localpath, String[] subfixs, Integer connectId) {
		List<String> fileNames = new ArrayList<>();
		List<FTPFile> fTPFiles = new ArrayList<>();
		// org.apache.commons.net.ftp
		FTPClient ftp = null;
		try {
			ftp = getFTPClient(hostname, username, password, port);
			// 主动模式
			ftp.enterLocalActiveMode();
			// 设置文件类型  二进制文件
			ftp.setFileType(FTPClient.BINARY_FILE_TYPE);
			ftp.setBufferSize(1*1024*1024);
			// 切换目录
			boolean flag = ftp.changeWorkingDirectory(new String(pathname.getBytes(), FTP.DEFAULT_CONTROL_ENCODING));
			if (!flag) {
				throw new Exception("目录[" + pathname + "]不存在");
			}
			FTPFile[] fs = ftp.listFiles(pathname, file -> {
				String name = file.getName();
				String strSubName = name.substring(name.lastIndexOf("."));
				for (String subfix : subfixs) {
					if (subfix.equalsIgnoreCase(strSubName)) {
						return true;
					}
				}
				return false;
			});
			for (FTPFile ftpFile : fs) {
				if(ftpFile.isFile()){
                    fTPFiles.add(ftpFile);
                }
			}
			//文件排序--时间早的文件优先处理
			Collections.sort(fTPFiles, (file, newFile) -> {
				if (file.getTimestamp().getTimeInMillis() < newFile.getTimestamp().getTimeInMillis()) {
					return -1;
				} else if (file.getTimestamp().getTimeInMillis() == newFile.getTimestamp().getTimeInMillis()) {
					return 0;
				} else {
					return 1;
				}
			});
			for (int i = 0; i < fTPFiles.size(); i++) {
				if (i >= DiConstants.FTP_FILE_COUNT_20s) {
					break;
				}
				FTPFile ff = fTPFiles.get(i);
				String strNewName = writeFile(ff, localpath, ftp, connectId);
				//deleteFile(hostname, port, username, password, pathname, strNewName);
				// 空文件判断
				if(StringUtils.isEmpty(strNewName)){
					break;
				}
				fileNames.add(strNewName);
			}
		} catch (Exception e) {
			logger.error("FTP下载文件异常{}", e);
		} finally {
			try {
				if (ftp != null) {
					ftp.logout();
				}
			} catch (IOException e1) {
				e1.printStackTrace();
			}
		
			if (ftp.isConnected()) {
				try {
					ftp.disconnect();
				} catch (IOException e) {
					e.printStackTrace();
				}
			}
		}
		return fileNames;
	}
	
	public static String writeFile(FTPFile ff, String localpath, FTPClient ftp, Integer connectId) {
		FileOutputStream out = null;
		InputStream in = null;
		String outFileName = ff.getName();
		String strConnect = "CONNECT";
		String strNewName = "";
		try {
			in = ftp.retrieveFileStream(new String(outFileName.getBytes("GB2312"),"ISO-8859-1"));
			if(in.available() > 0){
				strNewName = strConnect.concat(DiConstants.MARK_LEFT_BRACKET).concat(String.format("%03d", connectId)).concat(DiConstants.MARK_RIGHT_BRACKET).concat(outFileName);
				File localFile = new File(localpath.concat(File.separator).concat(strNewName));
				out = new FileOutputStream(localFile);
				
				byte[] byteArray = new byte[4096];
				int read = 0;
				while ((read = in.read(byteArray)) != -1) {
					out.write(byteArray, 0, read);
					out.flush();
				}
				out.close();
			}
			in.close();
			// 要多次操作这个ftp的流的通道,要等他的每次命令完成
			ftp.completePendingCommand();
		} catch (FileNotFoundException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		} finally {
			if (null != out) {
				try {
					out.close();
				} catch (IOException e) {
					e.printStackTrace();
				}
			}
			if (null != in) {
				try {
					in.close();
				} catch (IOException e) {
					e.printStackTrace();
				}
			}
		}
		return strNewName;
	}

 获取[in = ftp.retrieveFileStream(outFileName);]后判断再进行文件下载操作。如果不能读取文件内容，就停止本次操作。