利用线程池ThreadPoolExecutor生成多个文件，利用CountDownLatch来监控线程执行情况，然后利用FileChannel合并成一个文件，然后压缩上传到sftp

// 业务场景介绍 从hbase获取前七天所有车辆上的数据,并生成txt文件，之后压缩 上传到sftp文件服务器
// 利用线程池ThreadPoolExecutor生成多个文件，利用CountDownLatch来监控线程执行情况，然后利用 FileChannel合并成一个文件，然后压缩上传到sftp
// 最终前一天大约50g文件左右 压缩之后2.5g左右 耗时大约2.5小时

public void multiUpload(String startTime, String endTime,String type,String vehicleType) throws IOException, InterruptedException, ParseException {
    long begin= System.currentTimeMillis();
    String startDate;
    String endDate;
    List<String> pCodes= new ArrayList<>();
    if("1".equals(type)){
        //从mysql数据库中获取
        pCodes=businessWarningEmailMapper.getPcodeList(vehicleType);
        //获取自定义时间
        startDate=startTime;
        endDate=endTime;
        logger.error("****************Mysql-pcodeSize*************"+pCodes.size());
    } else {
        //从redis中获取
        pCodes = businessWarningEmailMapper.getPcodeList(vehicleType);
        Calendar calendar = Calendar.getInstance();
        calendar.add(Calendar.DATE, -7); //得到前七天
        Date date = calendar.getTime();
        startDate = df1.format(date) + "000000";
        endDate = df1.format(date) + "235959";
        logger.error("****************redis-pcodeSize*************"+pCodes.size());
    }
    //构造对象时候 需要传入参数N
    CountDownLatch doneSignal = new CountDownLatch(pCodes.size());
     //建立线程池 核心线程池 20个 最大线程池20个 线程最大空闲时间60s 线程等待队列
    ThreadPoolExecutor writeFilePool = new ThreadPoolExecutor(20, 20, 60, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<Runnable>(100000));
    for (String pCode : pCodes) {
        writeFilePool.submit(new Downloader(pCode, startDate, endDate, hBaseUtil, doneSignal));
    }
    doneSignal.await();
    //根据时间生成文件名
    Calendar calendar = Calendar.getInstance();
    Date date  ;
    if("1".equals(type)){
        date=formatTime.parse(startDate);
    }else {
        calendar.add(Calendar.DATE, -7); //得到前一天
        date = calendar.getTime();
    }
    //合并文件
     merge("/data/fileData/files","/data/fileData/zipData/"+df1.format(date) + ".txt");
     logger.error("合并文件完成");
    //压缩文件
    ZipUtil.zip("/data/fileData/zipData/" + df1.format(date) + ".txt", "/data/sourcedata/" + df1.format(date)+ ".zip");
    logger.error("压缩文件完成");
    //上传文件
    uploadToSftp(date);
    logger.error("上传文件完成");
    //删除临时文件
    deletefile("/data/fileData/files/");
    //FileUtil.del("/data/fileData/zipData/" + df1.format(date) + ".txt");
    logger.error("删除临时文件完成");
    logger.error("******************************耗时*************************"+((System.currentTimeMillis()-begin)/60000)+"分钟");

}


public static class Downloader implements Runnable {
    private static final String dir = "/data/fileData/files";
    private String pCode;
    private String startTime;
    private String endTime;
    private HBaseUtil hbaseUtil;
    private CountDownLatch doneSignal;
    public Downloader(String pCode, String startTime, String endTime, HBaseUtil hbaseUtil, CountDownLatch doneSignal) {
        this.pCode = pCode;
        this.startTime = startTime;
        this.endTime = endTime;
        this.hbaseUtil = hbaseUtil;
        this.doneSignal = doneSignal;
        File file = new File(dir);
        if (!file.exists()) {
            file.mkdir();
        }
    }
    @Override
    public void run() {
        Connection conn;
        try {
            conn = hbaseUtil.getConnection();
            TableName tableName = TableName.valueOf(Constants.TABLE_NAME);
            Table table = conn.getTable(tableName);
            ResultScanner scanner = null;
            Scan scan = new Scan();
            String pCodeMD5 = DigestUtils.md5DigestAsHex(pCode.getBytes());
            String startRow = pCodeMD5 + "-" + startTime;
            String stopRow = pCodeMD5 + "-" + endTime;
            scan.withStartRow(Bytes.toBytes(startRow), true);
            scan.withStopRow(Bytes.toBytes(stopRow), false);
            scan.setCaching(1000);
            scan.setCacheBlocks(false);
            scanner = table.getScanner(scan);
            int index = 0;
            File file = new File(dir + "/" + pCode + startTime + "-" + endTime + ".txt");
            BufferedWriter bw = new BufferedWriter(new FileWriter(file));
            StringBuffer sbr = new StringBuffer(5120000);
            logger.error("start downloader 1 : " + pCode + "#" + startRow + "#" + stopRow);
            for (Result result : scanner) {
                for (int i = 0; i < signals.length; i++) {
                    String signal = signals[i];
                    String flg = "";
                    byte[] res = result.getValue(signalFamily.getBytes(), signal.getBytes());
                    if (res != null) {
                        flg = Bytes.toString(res);
                        if (i == 0 || i == 1) {
                            flg = generate(flg, "dp5KEkDs19m7PSKI");
                        }
                    }
                    sbr.append(flg).append(",");
                }
                sbr.deleteCharAt(sbr.length() - 1).append(System.lineSeparator());
                index++;
                if (index % 1000 == 0) {
                    logger.error("start downloader 2 : " + sbr.length() + "#" + pCode);
                    bw.write(sbr.toString());
                    sbr = new StringBuffer(5120000);
                }
            }
            if (sbr.length() > 0) {
                logger.error("start downloader 3 : " + sbr.length() + "#" + pCode);
                bw.write(sbr.toString());
            }
            bw.close();
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            doneSignal.countDown();
        }
    }
}
/**
 sftp上传
 */
  private void uploadToSftp(Date date) {
    SFTPUtils sftp = null;
    sftp = new SFTPUtils(host, port, username, password);
    sftp.connect();
    logger.error("sftp开始上传");
    sftp.uploadFile(sftpUrl, df1.format(date) + ".zip", linuxUrl, df1.format(date)+ ".zip");
    logger.error("sftp上传完成");
}
/**
文件流关闭
*/
private void closeFile(FileOutputStream out) {
    if (out != null) {
        try {
            out.close();
        } catch (IOException e) {
            logger.error("关闭文件流失败！", e);
        }
    }
}

private void closeZipFile(ZipOutputStream zipOut) {
    if (zipOut != null) {
        try {
            zipOut.close();
        } catch (IOException e) {
            logger.error("关闭压缩流失败！", e);
        }
    }
}
/**
写文件
*/
public static class WriteFile implements Runnable {

    private String stringToSave;

    public WriteFile(String stringToSave) {
        this.stringToSave = stringToSave;
    }

    @Override
    public void run() {
        try {
            if (StringUtils.isNotBlank(stringToSave)) {
                zipOut.write(stringToSave.getBytes());
            }
        } catch (IOException e) {
            logger.error("Write to zip file info!", stringToSave);
        }
    }
}

/**
* 生成文件title文件用于合并
*/
public void createTitleFile() throws IOException {

    // 生成的文件路径
   String path = "/data/fileData/zipData/title.txt";
    File file = new File(path);
    if (!file.exists()) {
        file.getParentFile().mkdirs();
    }
    file.createNewFile();
    OutputStreamWriter fw = new OutputStreamWriter(new FileOutputStream(file), "UTF-8");
    BufferedWriter bw = new BufferedWriter(fw);
    StringBuffer sbr = new StringBuffer(5120000);
    for (String signal : signals) {
        sbr.append(signal).append(",");
    }
    sbr.deleteCharAt(sbr.length() - 1).append(System.lineSeparator());
    bw.write(sbr.toString());
    bw.flush();
    bw.close();
    fw.close();
}

/**
 * 合并多个文件
 *
 * @param from
 * @param to
 * @throws IOException
 */

public void merge(String from, String to) throws IOException {
    //生成title文件
    createTitleFile();
    File t = new File(to);
    FileInputStream in = null;
    FileChannel inChannel = null;

    FileOutputStream out = new FileOutputStream(t, true);
    FileChannel outChannel = out.getChannel();

    File f = new File(from);
    //写入title
    File titleFile = new File("/data/fileData/zipData/title.txt");
    in = new FileInputStream(titleFile);
    inChannel = in.getChannel();
    outChannel.transferFrom(inChannel, 0, titleFile.length());
    in.close();
    inChannel.close();
    // 获取目录下的每一个文件名，再将每个文件一次写入目标文件
    int num=0;
    if (f.isDirectory()) {
        File[] files = f.listFiles();
        logger.error("##########总数############"+files.length);
        // 记录新文件最后一个数据的位置
        long start = titleFile.length();
        for (File file : files) {
            if(file.length()>0){
                num++;
            }
            in = new FileInputStream(file);
            inChannel = in.getChannel();
            // 从inChannel中读取file.length()长度的数据，写入outChannel的start处
            outChannel.transferFrom(inChannel, start, file.length());
            start += file.length();
            in.close();
            inChannel.close();
        }
    }
    logger.error("##########上数据数量############"+num);
    out.close();
    outChannel.close();
}
/**
 * 对临时生成的文件夹和文件夹下的文件进行删除
 */
public static void deletefile(String delpath) {
    try {
        File file = new File(delpath);
        if (!file.isDirectory()) {
            file.delete();
        } else if (file.isDirectory()) {
            String[] filelist = file.list();
            for (int i = 0; i < filelist.length; i++) {
                File delfile = new File(delpath + File.separator + filelist[i]);
                if (!delfile.isDirectory()) {
                    delfile.delete();
                } else if (delfile.isDirectory()) {
                    deletefile(delpath + File.separator + filelist[i]);
                }
            }
            file.delete();
        }
    } catch (Exception e) {
        e.printStackTrace();
    }
}