干货-java HDFS系统上解压多文件zip压缩包

package com.zx.utils;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

import java.io.*;
import java.util.ArrayList;
import java.util.List;

import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream;
import org.apache.commons.compress.archivers.zip.*;

public class HdfsUnZip {
    public static void loadZipFileData(String hdfsFilePath) throws IOException {
        Configuration conf = new Configuration();
        String basePath = "hdfs://master:9000/";
        Path hdfs = new Path(basePath);
        //设置hdfs指定用户名root,否则没有权限访问hdfs
        System.setProperty("HADOOP_USER_NAME", "root");
        FileSystem fs = hdfs.getFileSystem(conf);

        FSDataInputStream fsInputStream = null;
        ZipArchiveInputStream zipInputStream = null;
        ZipArchiveEntry zipEntry = null;
        FSDataOutputStream mergerout = null;

        System.out.println(hdfsFilePath);

        try {
            //读取hdfs上文件到输入流
            fsInputStream = fs.open(new Path(hdfsFilePath));
            //将输入流转成字节数组
            byte[] b = toByteArray(fsInputStream);
            //去除字节数组中xx无用信息(实际为压缩包无用的头信息)
            b = offXXData(b);
            //将字节数组读入流,并创建zip处理流对象
            zipInputStream = new ZipArchiveInputStream( new ByteArrayInputStream(b));
            //判断是否能获取zipEntity对象
            while ((zipEntry = zipInputStream.getNextZipEntry()) != null) {
                //获取当前解压的文件名
                String entryName = zipEntry.getName();
                System.out.println("fileName:"+entryName);
                System.out.println(basePath+entryName);
                //判断解压文件是否非文件夹
                if (!zipEntry.isDirectory()) {
                    System.out.println("is file");
                    //在hdfs上创建指定文件
                    mergerout = fs.create(new Path(basePath+entryName));
                    int bygeSize1 = 10 * 1024 * 1024;
                    byte[] buffer1 = new byte[bygeSize1];
                    int nNumber;
                    //并将解压的内容写入hdfs文件
                    while ((nNumber = zipInputStream.read(buffer1, 0, bygeSize1)) != -1) {
                        mergerout.write(buffer1, 0, nNumber);
                    }
                    mergerout.flush();
                }else{
                    System.out.println("is Direcotry");
                }
            }
        } catch (Exception e) {
            e.printStackTrace();
        }finally {
            if(mergerout!=null){
                mergerout.close();
            }
            if(zipInputStream!=null){
                zipInputStream.close();
            }
            if(fsInputStream!=null){
                fsInputStream.close();
            }
        }
    }

    public static void zipHdfsFile(String hdfsFilePath) throws IOException {
        Configuration conf = new Configuration();
        String basePath = "hdfs://master:9000/";
        Path hdfs = new Path(basePath);
        //设置hdfs指定用户名root,否则没有权限访问hdfs
        System.setProperty("HADOOP_USER_NAME", "root");
        FileSystem fs = hdfs.getFileSystem(conf);

        FSDataInputStream fsInputStream = null;
        ZipArchiveInputStream zipInputStream = null;
        ZipArchiveEntry zipEntry = null;
        FSDataOutputStream mergerout = null;

        System.out.println(hdfsFilePath);

        try {
            //读取hdfs上文件到输入流
            fsInputStream = fs.open(new Path(hdfsFilePath));
            zipInputStream = new ZipArchiveInputStream(fsInputStream);
            //判断是否能获取zipEntity对象
            while ((zipEntry = zipInputStream.getNextZipEntry()) != null) {
                //获取当前解压的文件名
                String entryName = zipEntry.getName();
                System.out.println("fileName:"+entryName);
                System.out.println(basePath+"process/"+entryName);
                //判断解压文件是否非文件夹
                if (!zipEntry.isDirectory()) {
                    System.out.println("is file");
                    //在hdfs上创建指定文件
                    mergerout = fs.create(new Path(basePath+"process/"+entryName));
                    int bygeSize1 = 10 * 1024 * 1024;
                    byte[] buffer1 = new byte[bygeSize1];
                    int nNumber;
                    //并将解压的内容写入hdfs文件
                    while ((nNumber = zipInputStream.read(buffer1, 0, bygeSize1)) != -1) {
                        mergerout.write(buffer1, 0, nNumber);
                    }
                    mergerout.flush();
                }else{
                    System.out.println("is Direcotry");
                }
            }
        } catch (Exception e) {
            e.printStackTrace();
        }finally {
            if(mergerout!=null){
                mergerout.close();
            }
            if(zipInputStream!=null){
                zipInputStream.close();
            }
            if(fsInputStream!=null){
                fsInputStream.close();
            }
        }
    }

    public static void main(String[]args) throws IOException {
        zipHdfsFile("hdfs://master:9000/input.zip");
        loadZipFileData("hdfs://master:9000/input/2_74_1517985979_002_00_805690.zip");
    }


    /**
     * 获得去除XX后的数据
     *
     * @param data
     * @return
     * @throws IOException
     */
    public static byte[] offXXData(byte[] data) throws IOException {

        ByteArrayOutputStream out = null;
        CustomLineInputStream lis = null;

        try {
            out = new ByteArrayOutputStream();
            lis = new CustomLineInputStream(new ByteArrayInputStream(data));
            String line = null;
            List<Byte> bytes = new ArrayList<Byte>();

            boolean isFirstLine = true;
            while ((line = lis.readLineBytes(bytes)) != null) {

                if (isFirstLine && (!line.startsWith("XX-"))) {
                    return data;
                } else {
                    isFirstLine = false;
                }

                if (!line.startsWith("XX-")) {
                    break;
                }
                bytes.clear();

            }
            byte[] bs = new byte[1024];
            int i = 0, flag = 0;
            for (int j = 0; j < bytes.size(); j++) {
                byte b = bytes.get(j);
                if ((b == '\n' || b == '\r') && flag == 0) {
                    continue;
                } else {
                    flag++;
                }
                out.write(bytes.get(j));
            }

            while ((i = lis.read(bs)) != -1) {
                out.write(bs, 0, i);
            }

            out.flush();
            data = out.toByteArray();
        } catch (IOException e) {
            e.printStackTrace();
            throw e;
        } finally {
            if (out != null) {
                try {
                    out.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
            if (lis != null) {
                try {
                    lis.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
        return data;
    }

    //将文件读入流,然后换成byte数组
    public static byte[] File2byte(File file)
    {
        byte[] buffer = null;
        try
        {
            FileInputStream fis = new FileInputStream(file);
            ByteArrayOutputStream bos = new ByteArrayOutputStream();
            byte[] b = new byte[1024];
            int n;
            while ((n = fis.read(b)) != -1)
            {
                bos.write(b, 0, n);
            }
            fis.close();
            bos.close();
            buffer = bos.toByteArray();
        }
        catch (FileNotFoundException e)
        {
            e.printStackTrace();
        }
        catch (IOException e)
        {
            e.printStackTrace();
        }
        return buffer;
    }

    //输入流转成byte数组
    public static byte[] toByteArray(InputStream input) throws IOException {
        ByteArrayOutputStream output = new ByteArrayOutputStream();
        byte[] buffer = new byte[4096];
        int n = 0;
        while (-1 != (n = input.read(buffer))) {
            output.write(buffer, 0, n);
        }
        return output.toByteArray();
    }
}