java对大文件分片上传

• 这里记录一下，Java对大文件的切分，和后端接口分片上传的实现逻辑
  正常，前后端分离的项目其实是前端去切分文件，后端接口接收到切分后的分片文件去合并，这里都用java来记录一下。
• 特别说明：我这里用的是zip包的上传，里面是音频文件，如果你的文件是单个文件，切分和合并文件逻辑都是一样的，只是不用后续的解压。
• 因为是测试代码，所以部分代码规范不严谨

1.文件的切分

public static void chunkFile(){
        //每片的大小，这里是10M
        int TENMB = 10485760;
        //要切分的大文件和切分后的文件放到目录
       String PATH = "D:\\test\\fenpian\\";
        try {
           File file = new File(PATH, "55.zip");
            RandomAccessFile accessFile = new RandomAccessFile(file, "r");
            // 文件的大小
            long size = FileUtil.size(file);
            int chunkSize = (int) Math.ceil((double) size / TENMB);
            for (int i = 0; i < chunkSize; i++) {
                // 文件操作的指针位置
                long filePointer = accessFile.getFilePointer();
                byte[] bytes;
                if (i == chunkSize - 1) {
                    int len = (int) (size - filePointer);
                    bytes = new byte[len];
                    accessFile.read(bytes, 0, bytes.length);
                } else {
                    bytes = new byte[TENMB];
                    accessFile.read(bytes, 0, bytes.length);
                }
                FileUtil.writeBytes(bytes, new File(PATH, String.valueOf(i) + ".zip"));
            }
        } catch (FileNotFoundException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

2.Spring boot分片上传接口

controller层

	@Resource
    private FileUploadService fileUploadService;

    @RequestMapping(value = "/upload")
    public String upload(MultipartFileParam fileParam) throws IOException {
        try{
            return fileUploadService.fileUpload(fileParam);
        }catch (Exception e){
            e.printStackTrace();
            return "error";
        }
    }

service层

public interface FileUploadService {
    String fileUpload(MultipartFileParam fileParam) throws IOException;
 }

service实现层

@Slf4j
@Service
public class FileUploadServiceImpl implements FileUploadService {
	//合并后的文件的父目录
	private String FILE_UPLOAD_DIR = "D:\\test\\fenpian";
	//分片文件大小
	private Integer CHUNK_SIZE = 10485760;

	/**
     * 分片上传
     * @param fileParam
     * @return
     * @throws IOException
     */
    private String chunkUpload(MultipartFileParam fileParam) throws IOException {
        // 是否为最后一片
        boolean lastFlag = false;

        int currentChunk = fileParam.getChunk();
        int totalChunk = fileParam.getTotalChunk();
        long totalSize = fileParam.getTotalSize();
        String fileName = fileParam.getName();
        String fileMd5 = fileParam.getMd5();
        MultipartFile multipartFile = fileParam.getFile();

        String parentDir = FILE_UPLOAD_DIR + File.separator + fileMd5 + File.separator;
        String tempFileName = fileName + "_tmp";

        // 写入到临时文件
        File tmpFile = tmpFile(parentDir, tempFileName, multipartFile, currentChunk, totalSize, fileMd5);
        // 检测是否为最后一个分片（这里吧每个分片数据放到了一张表里，后续可以改为用redis记录）
        FileChunkRecordExample example = new FileChunkRecordExample();
        example.createCriteria().andMd5EqualTo(fileMd5);
        long count = fileChunkRecordMapper.countByExample(example);

        if (count == totalChunk) {
            lastFlag = true;
        }

        if (lastFlag) {
            // 检查md5是否一致
            log.info("是否最后一个分片:{}","是");
            if (!checkFileMd5(tmpFile, fileMd5)) {
                cleanUp(tmpFile, fileMd5);
                throw new RuntimeException("文件md5检测不符合要求, 请检查!");
            }
     
            System.out.println("开始重命名....");
            File newFile = renameFile(tmpFile, fileName);

            //解析文件数据 -解压缩unzip
            System.out.println("开始解压缩....");
            File zipFile = ZipUtil.unzip(newFile);
            //得到压缩包内所有文件

            System.out.println("遍历zipFile.....");
            File[] files = zipFile.listFiles();
            System.out.println("打印fileName.....");
            //解析文件，处理业务数据
            for (File file : files) {
                System.out.println(file.getName());
            }
            log.info("所有文件上传完成, 时间是:{}, 文件名称是:{}", DateUtil.now(), fileName);
			//所有数据都处理完成后，删除文件和数据库记录
            cleanUp(new File(parentDir + fileName),fileMd5);
        }else{
            log.info("是否最后一个分片:{}","否");
        }


        return "success";
    }

	private File tmpFile(String parentDir, String tempFileName, MultipartFile file,
                         int currentChunk, long totalSize, String fileMd5) throws IOException {
        log.info("开始上传文件, 时间是:{}, 文件名称是:{}", DateUtil.now(), tempFileName);
        long position = (currentChunk - 1) * CHUNK_SIZE;
        File tmpDir = new File(parentDir);
        File tmpFile = new File(parentDir, tempFileName);
        if (!tmpDir.exists()) {
            tmpDir.mkdirs();
        }

        RandomAccessFile tempRaf = new RandomAccessFile(tmpFile, "rw");
        if (tempRaf.length() == 0) {
            tempRaf.setLength(totalSize);
        }

        // 写入该分片数据
        FileChannel fc = tempRaf.getChannel();
        MappedByteBuffer map = fc.map(FileChannel.MapMode.READ_WRITE, position, file.getSize());
        map.put(file.getBytes());
        clean(map);
        fc.close();
        tempRaf.close();

        // 记录已经完成的分片
        FileChunkRecord fileChunkRecord = new FileChunkRecord();
        fileChunkRecord.setMd5(fileMd5);
        fileChunkRecord.setUploadStatus(1);
        fileChunkRecord.setChunk(currentChunk);
        fileChunkRecordMapper.insert(fileChunkRecord);
        log.info("分片文件上传完成, 时间是:{}, 文件名称是:{}", DateUtil.now(), tempFileName);
        return tmpFile;
    }

	private void cleanUp(File file, String md5) {
        if (file.exists()) {
            file.delete();
        }
        // 删除上传记录
        FileChunkRecordExample example = new FileChunkRecordExample();
        example.createCriteria().andMd5EqualTo(md5);
        fileChunkRecordMapper.deleteByExample(example);
    }


	/**
     * 最后一片接受完后执行
     * @param toBeRenamed
     * @param toFileNewName
     * @return
     */
    private File renameFile(File toBeRenamed, String toFileNewName) {
        // 检查要重命名的文件是否存在，是否是文件
        if (!toBeRenamed.exists() || toBeRenamed.isDirectory()) {
            log.info("File does not exist: " + toBeRenamed.getName());
            throw new RuntimeException("File does not exist");
        }
        String parentPath = toBeRenamed.getParent();
        File newFile = new File(parentPath + File.separatorChar + toFileNewName);
        // 如果存在, 先删除
        if (newFile.exists()) {
            newFile.delete();
        }
        toBeRenamed.renameTo(newFile);
        return newFile;
    }

	private static void clean(MappedByteBuffer map) {
        try {
            Method getCleanerMethod = map.getClass().getMethod("cleaner");
            Cleaner.create(map, null);
            getCleanerMethod.setAccessible(true);
            Cleaner cleaner = (Cleaner) getCleanerMethod.invoke(map);
            cleaner.clean();
        } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e) {
            e.printStackTrace();
        }
    }

/**
     * 文件md5值检查，最后一片文件合并后执行，
     * @param file    所有分片文件合并后的文件（正常情况下md5应该和前端传过来的大文件的md5一致）
     * @param fileMd5 大文件的md5值
     * @return
     * @throws IOException
     */
    private boolean checkFileMd5(File file, String fileMd5) throws IOException {
        FileInputStream fis = new FileInputStream(file);
        String checkMd5 = DigestUtils.md5DigestAsHex(fis).toUpperCase();
        fis.close();
        if (checkMd5.equals(fileMd5.toUpperCase())) {
            return true;
        }
        return false;
    }



	/**
     * 不分片
     * @param fileParam
     * @return
     */
    private String singleUpload(MultipartFileParam fileParam) {
        MultipartFile file = fileParam.getFile();
        File baseFile = new File(FILE_UPLOAD_DIR);
        if (!baseFile.exists()) {
            baseFile.mkdirs();
        }
        try {
            file.transferTo(new File(baseFile, fileParam.getName()));
            Date now = new Date();
            FileRecord fileRecord = new FileRecord();
            String filePath = FILE_UPLOAD_DIR + File.separator + fileParam.getName();
            long size = FileUtil.size(new File(filePath));
            String sizeStr = size / (1024 * 1024) + "Mb";
            fileRecord.setFileName(fileParam.getName()).setFilePath(filePath).setUploadStatus(1)
                    .setFileMd5(fileParam.getMd5()).setCreateTime(now).setUpdateTime(now).setFileSize(sizeStr);
            //fileRecordMapper.insert(fileRecord);
        } catch (IOException e) {
            log.error("单独上传文件错误, 问题是:{}, 时间是:{}", e.getMessage(), DateUtil.now());
        }
        return "success";
    }

}

后端入参实体类

package com.server.controller.bigFileUpload;

import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
import lombok.experimental.Accessors;
import org.springframework.web.multipart.MultipartFile;

/**
 * @description:
 * @date: created in 2021/10/6
 * @modified:
 */
@Getter
@Setter
@ToString
@Accessors(chain = true)
public class MultipartFileParam {
    /**
     * 是否分片
     */
    private boolean chunkFlag;

    /**
     * 当前为第几块分片
     */
    private int chunk;

    /**
     * 总分片数量
     */
    private int totalChunk;

    /**
     * 文件总大小, 单位是byte
     */
    private long totalSize;

    /**
     * 文件名
     */
    private String name;

    /**
     * 文件
     */
   
    private MultipartFile file;

    /**
     * md5值
     */
    private String md5;

}

合并后的文件放到了文件mdf的文件夹内