文件分片上传案例（FTP）

大文件分片上传，在上传过程中，如果出现网络异常导致文件块上传失败时，可以重新上传失败的文件块，而不用上传整个大文件，本文介绍文件分片上传到FTP服务器的案例。

思路

• 分片上传前根据文件大小和分片大小计算文件块数量，开启一个分片上传任务
• 将大文件分成若干块，每个文件块有两个关键参数
  • 任务ID
  • 文件块序号
• 将文件块存储到临时目录
• 当服务端接收到最后一个文件块时合并文件

实现代码

用本地内存缓存任务，FTP存储临时文件，单节点的话临时存到硬盘上也是可以的，实现的思路是一样的。

实体类

// 任务
class ShardingTask {

    /**
     * 任务ID
     * 用于关联分片
     */
    private String taskId;
    
    /**
     * 目的路径
     */
    private String destPath;

    /**
     * 文件名
     */
    private String filename;

    /**
     * 分片数量
     */
    private int number;

}

// 分块文件
class ShardingFile {

    /**
     * 任务ID
     * 用于关联分片
     */
    private String taskId;

    /**
     * 分片序号
     * from 0
     */
    private int serial;

}

Service

// 接口定义
interface ShardingFileInterface {

	/**
	 * 多节点修改成 redis 或 db 等存储
	 */
    Map<String, ShardingTaskVo> SHARDING_MAP = new HashMap<>();

	/**
	 * 分片临时目录
	 */
    String SHARDING_PATH = "/sharding/";

    /**
     * 开始任务
     * @param shardingTask
     */
    void startTask(ShardingTaskVo shardingTask);

    /**
     * 分片上传文件
     * @param shardingFile
     * @param inputStream   close
     */
    void uploadFile(ShardingFileVo shardingFile, InputStream inputStream);

}

// 实现类
public class ShardingFileService implements ShardingFileInterface {

    private final FtpClient ftpClient ;

    @SneakyThrows
    @Override
    public void startTask(ShardingTaskVo shardingTask) {
        changeDirectory(SHARDING_PATH);
        // 每个任务创建一个目录
        ftpClient.makeDirectory(shardingTask.getTaskId());
        // 缓存任务
        SHARDING_MAP.put(shardingTask.getTaskId(), shardingTask);
    }

    @SneakyThrows
    @Override
    public void uploadFile(ShardingFileVo shardingFile, InputStream inputStream) {
        changeDirectory(SHARDING_PATH + shardingFile.getTaskId());

        // 写文件
        ftpClient.putFile(String.valueOf(shardingFile.getSerial()), inputStream);

        // 判断当前文件是否为最后一块
        if (!lastOne(shardingFile.getTaskId())) {
            return;
        }

        genFileAndUpload(shardingFile);
    }

    /**
     * 判断是否最后一块
     * @param taskId
     * @return
     */
    @SneakyThrows
    private boolean lastOne(String taskId) {
        int len = 1;
        Iterator<FtpDirEntry> iterator = ftpClient.listFiles("");
        while (iterator.hasNext() && iterator.next() != null) {
            len++;
        }
        ShardingTaskVo shardingTask = SHARDING_MAP.get(taskId);
        if (null == shardingTask) {
            return false;
        }
        return shardingTask.getNumber() == len;
    }

     @SneakyThrows
    private void genFileAndUpload(ShardingFileVo shardingFile) {
        ShardingTaskVo shardingTask = SHARDING_MAP.get(shardingFile.getTaskId());

        int number = shardingTask.getNumber();

        String remoteFile = shardingTask.getDestPath() + shardingTask.getFilename();

        try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) {
            for (int i = 1; i <= number; i++) {
                try (InputStream inputStream = ftpClient.getFileStream(String.valueOf(i));) {
                    write(outputStream, inputStream);
                    log.info("write: {}", i);
                }
            }
            ftpClient.putFile(remoteFile, new ByteArrayInputStream(outputStream.toByteArray()));
        } finally {
            // 删除文件块
            removeDirectoryAndFiles(SHARDING_PATH + shardingTask.getTaskId());
        }

    }


    /**
     * 写到输出流
     * @param outputStream
     * @param inputStream
     */
    @SneakyThrows
    private void write(OutputStream outputStream, InputStream inputStream) {
        byte[] bytes = new byte[1024];
        while (inputStream.read(bytes) > 0) {
            outputStream.write(bytes);
        }
    }


    /**
     * 切换目录
     * 不存在时自动创建目录
     * @param remoteDirectory
     */
    @SneakyThrows
    private void changeDirectory(String remoteDirectory) {
        try {
            ftpClient.changeDirectory(remoteDirectory);
        } catch (Exception e) {
            ftpClient.makeDirectory(remoteDirectory);
            ftpClient.changeDirectory(remoteDirectory);
        }
    }

    /**
     * 删除目录及其文件
     * @param remoteDirectory
     */
    @SneakyThrows
    private void removeDirectoryAndFiles(String remoteDirectory) {
        changeDirectory(remoteDirectory);
        Iterator<FtpDirEntry> iterator = ftpClient.listFiles("");
        while (iterator.hasNext()) {
            FtpDirEntry next = iterator.next();
            ftpClient.deleteFile(next.getName());
        }
        ftpClient.removeDirectory(remoteDirectory);
    }

}

Demo

	@Autowired
    private ShardingFileInterface shardingFileInterface;

    @Test
    public void uploadFile() throws Exception {
        ShardingTaskVo taskVo = new ShardingTaskVo();
        taskVo.setTaskId(UUID.randomUUID().toString());
        taskVo.setFilename("xxx.png");
        taskVo.setDestPath("/img/");
        taskVo.setMd5("");

        try (InputStream inputStream = Files.newInputStream(Paths.get("/Users/waani/xxx.png"));) {

            // 分片大小 10k
            int sharding = 10240;
			
			// 文件大小
            int available = inputStream.available();
			
			// 分片数量
            int number = (available / sharding) + 1;
            taskVo.setNumber(number);	
            
            // 开启任务
            shardingFileInterface.startTask(taskVo);

            byte[] bytes = new byte[sharding];
            ShardingFileVo fileVo = new ShardingFileVo();
            fileVo.setTaskId(taskVo.getTaskId());
            int i = 1;
            while (inputStream.read(bytes) > 0) {
                ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(bytes);
                fileVo.setSerial(i++);
                // 分片上传
                shardingFileInterface.uploadFile(fileVo, byteArrayInputStream);
            }
        }

    }

源码