大数据传输-文件分段传输

文件分段传输,按照我的理解就是,将大文件分成多个小文件进行传输,因为数据量大,在传输过程中难免会中断,所以在中断后也可进行续传,下面是我手写的分段传输代码

文件推送:

  /**
     * 文件推送
     *
     * @return
     * @throws Exception
     */
    public synchronized boolean pushCategoryEntity() throws Exception {
        long start = System.currentTimeMillis();
        //此文件为存储当前成功传输的数据量,传输出错是可接着再来,可用redis存储
        File fw = new File("******/pushnode", "node-****.txt");
        int pageNum = 1;
        int pageSize = 10;
        //分段传输记录传输节点信息
        BufferedWriter bw = null;
        List<CategoryEntity> categoryEntities = null;
        BufferedWriter bos = null;
        String proKey = "加密秘钥自定义设置";
        //获取要传输数据总条数
        int count = this.getCount("sql");
        try {
            //调用推送到指定服务器接口
            Map<String, String> map = new HashMap<>();
            //可设置除文件外的推送信息map
            String url = "http://127.0.0.1:10001/product/category/uploadFile";
            if (count > 1000) {
                if (fw.exists()) {
                    //fw文件存在则读取上传传输位置的指针
                    BufferedReader br = null;
                    try {
                        br = new BufferedReader(new FileReader(fw));
                        pageNum = Integer.parseInt(br.readLine());
                    } catch (NumberFormatException e) {
                        if (br != null)
                            br.close();
                        fw.delete();
                        e.printStackTrace();
                    } finally {
                        if (br != null)
                            br.close();
                    }
                }

                bo:
                //设置每次推送1000条,通过分页控制
                for (int i = 0; i <= count / 1000; i++) {
                    File file = new File("******/pushfile", i + "push-****.txt");
                    if (file.exists()) {
                        file.delete();
                    }
                    bos = new BufferedWriter(new FileWriter(file));
                    while (pageNum % 100 != 0) {
                        categoryEntities = dao.getCategoryEntity(pageNum, pageSize);
                        pageNum += 1;
                        if (CollectionUtils.isEmpty(categoryEntities)) {
                            break bo;
                        }
                        String json = JSONObject.toJSONString(categoryEntities);
                        //文件加密(可自定义加密工具)
                        String encrypt = Encryp.getInstance().encrypt(json, Encryp.SM100, proKey);
                        bos.write(encrypt);
                        bos.newLine();
                        bos.flush();
                    }
                    categoryEntities = dao.getCategoryEntity(pageNum, pageSize);
                    pageNum += 1;
                    if (CollectionUtils.isEmpty(categoryEntities)) {
                        break;
                    }
                    String json = JSONObject.toJSONString(categoryEntities);
                    //文件加密
                    String encrypt = Encryp.getInstance().encrypt(json, Encryp.SM100, proKey);
                    bos.write(encrypt);
                    bos.newLine();
                    bos.flush();
                    //使用md5工具类校验文件完整性
                    MultipartFile multipartFile = getMultipartFile(file);
                    map.put("fileMd5", Md5Test.getFileMD5String(multipartFile));
                    String result = HttpUtils.doPostFilePlus(url, map, multipartFile, StandardCharsets.UTF_8);
                    //返回结果校验,我就尽量简单了
                    if (!(result != null && result.contains("success"))) {
                        return false;
                    } else {
                        bw = new BufferedWriter(new FileWriter(fw, false));
                        bw.write(String.valueOf(pageNum));
                        bw.flush();
                        bw.close();
                        bos.close();
                        file.delete();
                    }
                }
                if (fw.exists()) {
                    if (bw != null) {
                        bw.close();
                    }
                    fw.delete();
                }
                long end = System.currentTimeMillis();
                logger.info("推送: 耗时: " + (end - start));
                return true;
            } else {
                //数据小于1000条就不进行分段推送了
                File file = new File("******/pushfile", "push-****.txt");
                if (file.exists()) {
                    file.delete();
                }
                bos = new BufferedWriter(new FileWriter(file));
                while (true) {
                    categoryEntities = dao.getCategoryEntity(pageNum, pageSize);
                    pageNum += 1;
                    if (CollectionUtils.isEmpty(categoryEntities)) {
                        break;
                    }
                    String json = JSONObject.toJSONString(categoryEntities);
                    //文件加密(可自定义加密工具)
                    String encrypt = Encryp.getInstance().encrypt(json, Encryp.SM100, proKey);
                    bos.write(encrypt);
                    bos.newLine();
                    bos.flush();
                }
                //使用md5工具类校验文件完整性
                MultipartFile multipartFile = getMultipartFile(file);
                map.put("fileMd5", Md5Test.getFileMD5String(multipartFile));
                String result = HttpUtils.doPostFilePlus(url, map, multipartFile, StandardCharsets.UTF_8);
                if (result != null && result.contains("success")) {
                    bos.close();
                    file.delete();
                    long end = System.currentTimeMillis();
                    logger.info("推送: 耗时: " + (end - start));
                    return true;
                }
            }
        } catch (Exception e) {
            //自定义异常
            throw new BaseException("推送失败", e.getMessage());
        } finally {
            if (bos != null) {
                bos.close();
            }
            if (bw != null) {
                bw.close();
            }
        }
        return false;
    }

controller接收格式:

@RequestParam("file") MultipartFile file, @RequestParam Map<String, String> params

 接收传输文件:

 @Transactional(propagation = Propagation.REQUIRES_NEW)
    public ResPushSgccData acceptCategoryEntity(MultipartFile file, Map<String, String> map) throws Exception {
        // 接收网省推送的文件,并存储
        BufferedReader br = null;
        try {
            // 文件完整性校验
            String fileMd5 = Md5Test.getFileMD5String(file);
            System.out.println(fileMd5);
            String fileMd51 = map.get("fileMd5");
            if (!fileMd5.equals(fileMd51)){
                //自定义异常
                throw new BaseException("文件完整性被破坏!");
            }

            br = new BufferedReader(new InputStreamReader(file.getInputStream()));
            String proKey = "加密秘钥自定义设置";
            String line;
            while ((line = br.readLine()) != null) {
                // 文件解密(自定义加解密)
                String decrypt = Encryp.getInstance().decrypt(line, Encryp.SM100, proKey);
                List<CategoryEntity> categoryEntities =
                        JSONArray.parseArray(decrypt, CategoryEntity.class);
                //保存到库
                dao.batchAdd(categoryEntities);
            }
            return ResPushSgccData.ok();
        } catch (Exception e) {
            throw new BaseException("接受失败", e.getMessage());
        } finally {
            if (br != null) {
                br.close();
            }
        }
    }