Java 并发写文件加锁处理(阿里云OSS HTTP文件迁移)

目录

文件追加的三种方式

OSS HTTP list.txt输出方式

---

阿里云OSS文件资源行数据迁移：

https://help.aliyun.com/document_detail/95134.html?spm=a2c4g.11186623.6.555.32755641l0V8YD

文件追加的三种方式

文件多线程写的情况下需要加锁处理： 提供三种方式写入文件内容。

 /**
     * 追加写入内容到目标文件
     *
     * @param dwCode
     * @param rowsContent
     * @param destFile
     */
    private synchronized void writeIntoFile(String dwCode,String rowsContent,File destFile){
        log.info("writeIntoFile {} dts data......", dwCode);
        OutputStream os = null;
        try {
            if (!destFile.exists()) {
                destFile.createNewFile();
            }
            // 追加写入内容
            os = new FileOutputStream(destFile,true);
            os.write(rowsContent.getBytes());
            log.info("writeIntoFile {} dts data......success!", dwCode);
        } catch (FileNotFoundException e) {
            log.info("writeIntoFile {} dts data......FileNotFoundException!", dwCode);
            e.printStackTrace();
        } catch (IOException e) {
            log.info("writeIntoFile {} dts data......IOException!", dwCode);
            e.printStackTrace();
        } finally {
            if (null != os) {
                try {
                    os.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
    }

    /**
     * 追加写入内容到目标文件
     *
     * @param dwCode
     * @param rowsContent
     * @param destFile
     */
    private void writeIntoFileByBufferedWriter(String dwCode,String rowsContent,File destFile){
        log.info("writeIntoFileByBufferedWriter {} dts data......", dwCode);
        BufferedWriter out = null;
        try {
            if (!destFile.exists()) {
                destFile.createNewFile();
            }
            // 追加写入内容
            out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(destFile, true)));
            out.write(rowsContent);
            log.info("writeIntoFileByBufferedWriter {} dts data......success!", dwCode);
        } catch (FileNotFoundException e) {
            log.info("writeIntoFileByBufferedWriter {} dts data......FileNotFoundException!", dwCode);
            e.printStackTrace();
        } catch (IOException e) {
            log.info("writeIntoFileByBufferedWriter {} dts data......IOException!", dwCode);
            e.printStackTrace();
        } finally {
            if(null != out){
                try {
                    out.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
    }

    /**
     * 追加写入内容到目标文件
     *
     * @param dwCode
     * @param rowsContent
     * @param destFile
     */
    private synchronized void writeIntoFileByFileWriter(String dwCode,String rowsContent,File destFile){
        log.info("writeIntoFileByAppend {} dts data......", dwCode);
        OutputStream os = null;
        try {
            if (!destFile.exists()) {
                destFile.createNewFile();
            }
            // 追加写入内容
            FileWriter fileWriter =new FileWriter(destFile,true);
            fileWriter.write(rowsContent);
            log.info("writeIntoFileByAppend {} dts data......success!", dwCode);
        } catch (FileNotFoundException e) {
            log.info("writeIntoFileByAppend {} dts data......FileNotFoundException!", dwCode);
            e.printStackTrace();
        } catch (IOException e) {
            log.info("writeIntoFileByAppend {} dts data......IOException!", dwCode);
            e.printStackTrace();
        } finally {
            if (null != os) {
                try {
                    os.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
    }

OSS HTTP list.txt输出方式

pom.xml

  <dependency>
            <groupId>com.aliyun.oss</groupId>
            <artifactId>aliyun-sdk-oss</artifactId>
            <version>3.8.0</version>
        </dependency>

Java 代码

package com.aliyun.service.oss.tds;

import com.aliyun.service.oss.service.BaseService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import java.io.*;
import java.util.List;
import java.util.Map;

/**
 * @Copyright: 2019-2021
 * @FileName: JilinFileService.java
 * @Author: PJL
 * @Date: 2020/6/10 17:02
 * @Description: **文件服务
 */
@Slf4j
@Service
public class FileService extends BaseService {

    @Value("${system.http-base-url}")
    String httpBaseUrl;

    @Value("${system.local-save-base-path}")
    String localSaveBasePath;

    @Autowired
    FujianService fujianService;

    /**
     * 解析文件路径
     *
     * @param map
     * @param sb
     * @param fieldName
     */
    private String parseFilePath(Map<String, Object> map,StringBuffer sb,String fieldName){
        Object value = map.get(fieldName);
        if(null == value){
            return "";// 无值情况不予处理
        }
        String file = value.toString();
        String httpUrl = new StringBuffer(httpBaseUrl).append("/").append(file).toString();
        //String destUrl = new StringBuffer(dwCode).append("/").append(file).toString();
        String destUrl = new StringBuffer(file).toString();
        String rowContent = new StringBuffer(httpUrl).append("\t").append(destUrl).append("\n").toString();
        sb.append(rowContent);
        return destUrl;
    }

    /**
     * 解析并封装文件路径集合字符串内容
     *
     * @param dwCode
     * @param dataList
     */
    private String parseDataListContent(String dwCode, List<Map<String, Object>> dataList) {
        StringBuffer sb = new StringBuffer();
        if (null != dataList && dataList.size() > 0) {
            for (Map<String, Object> map : dataList) {
                // 原文件解析
                String destUrl = this.parseFilePath(map,sb,"FJ_LJ");
                // 缩略图解析
                if(StringUtils.isNotEmpty(destUrl) && !destUrl.contains(".mp4")){
                    this.parseFilePath(map,sb,"FJ_LJ_COMPRES");
                }
            }
        }
        return sb.toString();
    }

    /**
     * 创建路径
     *
     * @param filePath
     */
    private void createDir(String filePath) {
        File dwDirFile = new File(filePath);
        if (!dwDirFile.exists()) {
            dwDirFile.mkdir();
        }
    }

    /**
     * 追加写入内容到目标文件
     *
     * @param dwCode
     * @param rowsContent
     * @param destFile
     */
    private synchronized void writeIntoFile(String dwCode,String rowsContent,File destFile){
        log.info("writeIntoFile {} dts data......", dwCode);
        OutputStream os = null;
        try {
            if (!destFile.exists()) {
                destFile.createNewFile();
            }
            // 追加写入内容
            os = new FileOutputStream(destFile,true);
            os.write(rowsContent.getBytes());
            log.info("writeIntoFile {} dts data......success!", dwCode);
        } catch (FileNotFoundException e) {
            log.info("writeIntoFile {} dts data......FileNotFoundException!", dwCode);
            e.printStackTrace();
        } catch (IOException e) {
            log.info("writeIntoFile {} dts data......IOException!", dwCode);
            e.printStackTrace();
        } finally {
            if (null != os) {
                try {
                    os.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
    }

    /**
     * 追加写入内容到目标文件
     *
     * @param dwCode
     * @param rowsContent
     * @param destFile
     */
    private void writeIntoFileByBufferedWriter(String dwCode,String rowsContent,File destFile){
        log.info("writeIntoFileByBufferedWriter {} dts data......", dwCode);
        BufferedWriter out = null;
        try {
            if (!destFile.exists()) {
                destFile.createNewFile();
            }
            // 追加写入内容
            out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(destFile, true)));
            out.write(rowsContent);
            log.info("writeIntoFileByBufferedWriter {} dts data......success!", dwCode);
        } catch (FileNotFoundException e) {
            log.info("writeIntoFileByBufferedWriter {} dts data......FileNotFoundException!", dwCode);
            e.printStackTrace();
        } catch (IOException e) {
            log.info("writeIntoFileByBufferedWriter {} dts data......IOException!", dwCode);
            e.printStackTrace();
        } finally {
            if(null != out){
                try {
                    out.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
    }

    /**
     * 追加写入内容到目标文件
     *
     * @param dwCode
     * @param rowsContent
     * @param destFile
     */
    private synchronized void writeIntoFileByFileWriter(String dwCode,String rowsContent,File destFile){
        log.info("writeIntoFileByAppend {} dts data......", dwCode);
        OutputStream os = null;
        try {
            if (!destFile.exists()) {
                destFile.createNewFile();
            }
            // 追加写入内容
            FileWriter fileWriter =new FileWriter(destFile,true);
            fileWriter.write(rowsContent);
            log.info("writeIntoFileByAppend {} dts data......success!", dwCode);
        } catch (FileNotFoundException e) {
            log.info("writeIntoFileByAppend {} dts data......FileNotFoundException!", dwCode);
            e.printStackTrace();
        } catch (IOException e) {
            log.info("writeIntoFileByAppend {} dts data......IOException!", dwCode);
            e.printStackTrace();
        } finally {
            if (null != os) {
                try {
                    os.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
    }

    /**
     * 写入同一个文件
     *
     * @param dwCode
     * @param rowsContent
     */
    private  void saveToSingleFile(String dwCode, String rowsContent) {
        log.info("save saveToSingleFile {} dts data......", dwCode);
        String filePath = new StringBuffer(localSaveBasePath).append("//").append("list.txt").toString();
        File destFile = new File(filePath);
        this.writeIntoFile(dwCode,rowsContent,destFile);
    }

    /**
     * 保存单位附件文件
     *
     * @param type
     * @param dwCode
     * @param rowsContent
     */
    private synchronized void saveToMultiFile(String type, String dwCode, String rowsContent) {
        if (StringUtils.isNotEmpty(rowsContent)) {
            /创建路径//S/
            String dwDir = new StringBuffer(localSaveBasePath).append("//").append(type).toString();
            this.createDir(dwDir);
            String dwDir2 = new StringBuffer(dwDir).append("//").append(dwCode).toString();
            this.createDir(dwDir2);
            /创建路径//E/
            String destFilePath = new StringBuffer(dwDir2).append("//").append("list.txt").toString();
            File destFile = new File(destFilePath);
            this.writeIntoFile(dwCode,rowsContent,destFile);
        }
    }

    /**
     * 保存病疫监测事件附件
     *
     */
    private void saveByjcEventFujianAll(boolean writeSingleFile) {
        List<String> list = fujianService.getExportCorpList();
        log.info("ByjcEvent getExportCorpList  size = {}", list.size());
        for (String dwCode : list) {
            List<Map<String, Object>> dataList = fujianService.getEventFujian(FujianService.XH_BYJC_FJ_TB, dwCode);
            // 解析并保存病疫监测附件数据
            String rowsContent = this.parseDataListContent(dwCode, dataList);
            // 保存迁移文件内容
            if(writeSingleFile){
                this.saveToSingleFile(dwCode, rowsContent);
            }else{
                this.saveToMultiFile(FujianService.XH_BYJC_FJ_TB, dwCode, rowsContent);
            }
        }
        log.info("ByjcEvent getExportCorpList  size = {}   finished!", list.size());
    }

    /**
     * 保存公共事件附件
     *
     */
    private void saveShijianEventFujianAll(boolean writeSingleFile) {
        List<String> list = fujianService.getExportCorpList();
        log.info("ShijianEvent getExportCorpList  size = {}", list.size());
        for (String dwCode : list) {
            List<Map<String, Object>> dataList = fujianService.getEventFujian(FujianService.XH_SHIJIAN_FJ_TB, dwCode);
            // 解析并保存病疫监测附件数据
            String rowsContent = this.parseDataListContent(dwCode, dataList);
            // 保存迁移文件内容
            if(writeSingleFile){
                this.saveToSingleFile(dwCode, rowsContent);
            }else{
                this.saveToMultiFile(FujianService.XH_SHIJIAN_FJ_TB, dwCode, rowsContent);
            }
        }
        log.info("ShijianEvent getExportCorpList  size = {}   finished!", list.size());
    }

    /**
     * 开启线程处理附件
     *
     */
    public void startTDSStoreThreads(boolean writeSingleFile) {
        log.info("start threads.....");
        new Thread(() -> {
            this.saveByjcEventFujianAll(writeSingleFile);
        }).start();

        new Thread(() -> {
            this.saveShijianEventFujianAll(writeSingleFile);
        }).start();
        log.info("start threads.....success!");
    }
}