hdfs操作类

ManagerHdfsUtil.class

package cn.getech.data.manager.utils;

import cn.getech.data.development.constant.DataDevelopmentBizExceptionEnum;
import cn.getech.data.intelligence.common.exception.RRException;
import cn.getech.data.manager.config.ManagerDataConfig;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.log4j.Logger;

import javax.servlet.http.HttpServletResponse;
import java.io.*;
import java.net.URI;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.UUID;

/**
 * hdfs工具类
 */
public class ManagerHdfsUtil {
    private static Logger logger = Logger.getLogger(ManagerHdfsUtil.class);
    public DistributedFileSystem dfs;
    public Configuration conf;

    public ManagerHdfsUtil(ManagerDataConfig managerDataConfig) throws Exception {
//        Configuration conf=new Configuration(false);
        conf = new Configuration(false);
        String ns = managerDataConfig.getNamespace();
        String[] nameNodesAddr = managerDataConfig.getNamenodestr().split(",");
        String[] nameNodes = {"nn1", "nn2"};
        conf.set("fs.defaultFS", "hdfs://" + ns);
        conf.set("dfs.nameservices", ns);
        conf.set("dfs.ha.namenodes." + ns, nameNodes[0] + "," + nameNodes[1]);
        conf.set("dfs.namenode.rpc-address." + ns + "." + nameNodes[0], nameNodesAddr[0]);
        conf.set("dfs.namenode.rpc-address." + ns + "." + nameNodes[1], nameNodesAddr[1]);
        conf.set("dfs.client.failover.proxy.provider." + ns, "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider");
        conf.set("dfs.client.use.datanode.hostname", "true");
        conf.set("dfs.socket.timeout", "6000000");
        //路径权限
        conf.set("fs.permissions.umask-mode","000");
        String hdfsRPCUrl = "hdfs://" + ns + ":" + managerDataConfig.getNameport();
        dfs = new DistributedFileSystem();
        dfs.initialize(URI.create(hdfsRPCUrl), conf);
    }


    /**
     * 获取输出流
     *
     * @param dst         目标路径
     * @return 成功还是失败
     */
    public FSDataOutputStream writeFile( String dst) {
        Path dstPath = new Path(dst);
        try {
            FSDataOutputStream outputStream;

            if (dfs.exists(dstPath)) {
                dfs.delete(dstPath, true);
            }
            if (!dfs.exists(dstPath.getParent())) {
                dfs.mkdirs(dstPath.getParent());
            }
            //Init output stream
            outputStream = dfs.create(dstPath);
            return outputStream;
        } catch (IOException ie) {
            logger.error(ie.getMessage());
            throw new RRException("获取流失败");
        }
    }


    /**
     * 上传方法
     *
     * @param fileContent 内容
     * @param dst         目标路径
     * @return 成功还是失败
     */
    public boolean writeFile(byte[] fileContent, String dst) {
        Path dstPath = new Path(dst);
        try {
            if (dfs.exists(dstPath)) {
                dfs.delete(dstPath, true);
            }
            if (!dfs.exists(dstPath.getParent())) {
                dfs.mkdirs(dstPath.getParent());
            }
            //Init output stream
            FSDataOutputStream outputStream = dfs.create(dstPath);
            //Cassical output stream usage
            outputStream.write(fileContent);
            outputStream.close();
        } catch (IOException ie) {
            logger.error(ie.getMessage());
            return false;
        }
        return true;
    }




    /**
     * 上传方法
     *
     * @param fileContent 内容
     * @param dst         目标路径
     * @param partitions  分区路径
     * @param writeType   追加方式 2:追加  1：覆盖
     * @return 成功还是失败
     */
    public boolean writeFile(byte[] fileContent, String dst, String partitions, String fileName, Integer writeType) {
        String rodownDataDst = dst + "/" + fileName + "&&&&" + UUID.randomUUID();
        if (StringUtils.isNotEmpty(partitions)) {
            rodownDataDst = dst + "/" + partitions + fileName + "&&&&" + UUID.randomUUID();
            dst = dst + "/" + partitions;
        }
        Path dstPath = new Path(dst);
        try {
            if (Objects.equals(2, writeType)) {
                //追加
            } else {
                //覆盖
                if (dfs.exists(dstPath)) {
                    dfs.delete(dstPath, true);
                }
            }
            dstPath = new Path(rodownDataDst);
            if (!dfs.exists(dstPath.getParent())) {
                //创建777的目录，防止其他用户访问hive的数据有权限问题
                dfs.mkdirs(dstPath.getParent(),new FsPermission(FsAction.ALL,FsAction.ALL,FsAction.ALL));
            }
            //Init output stream
            FSDataOutputStream outputStream = dfs.create(dstPath);
            //Cassical output stream usage
            outputStream.write(fileContent);
            outputStream.close();
        } catch (IOException ie) {
            logger.error(ie.getMessage());
            return false;
        }
        return true;
    }




    /**
     * @param hdfsUrl 绝对路径
     * @return
     * @throws IOException
     */
    //TODO 改为返回流格式
    public BufferedInputStream readFile(String hdfsUrl) {
        FSDataInputStream fis = null;
//        Configuration conf = getConf();
        Path path = new Path(hdfsUrl);
        BufferedInputStream bis = null;
        try {
            FileSystem fs = FileSystem.get(URI.create(hdfsUrl), conf);

            if (fs.exists(path)) {
                fis = fs.open(path);
                bis = new BufferedInputStream(fis);


//                fis = fs.open(path);
//                byte[] buffer = new byte[fis.available()];
//                fis.read(buffer);
//                fis.close();
//                // 清空response
//                response.reset();
//                // 设置response的Header
//                response.addHeader("Content-Disposition", "attachment;filename=" + primaryFileName);
                response.addHeader("Content-Length", "" + file.length());
//                OutputStream toClient = new BufferedOutputStream(response.getOutputStream());
//                response.setContentType("application/octet-stream");
//                toClient.write(buffer);
//                toClient.flush();
//                toClient.close();
            } else {
                throw new RRException(DataDevelopmentBizExceptionEnum.MODEL_NOT_FOUND_IN_HDFS.getMessage());
            }
        } catch (IOException | IllegalArgumentException e) {
            logger.error(e.getMessage());
        }
        return bis;
//        finally {
//            try{
//                outputStream.flush();
//                outputStream.close();
//                fis.close();
//            }catch (IOException e){
//                e.printStackTrace ();
//            }
//        }
    }




    /**
     * 获取路径下所有文件的路径进而获取table名字
     *
     * @param dir
     * @throws IOException
     */
    public List<String> dbTableNameList(String dir) throws IOException {
//        Configuration conf = getConf();
        FileSystem fs = FileSystem.get(conf);
        FileStatus[] stats = null;
        List<String> tableNameList = new ArrayList<>();
        //判断路径是否存在
        if (fs.exists(new Path(dir))) {
            stats = fs.listStatus(new Path(dir));
            for (int i = 0; i < stats.length; i++) {
                if (stats[i].isDirectory()) {
                    tableNameList.add(stats[i].getPath().getName());
                } else {
                    System.out.println(stats[i].getPath().toString());
                }
            }
            fs.close();
        }
        return tableNameList;
    }


    public boolean delete(String dst) {
        Path dstPath = new Path(dst);
        try {
            if (dfs.exists(dstPath)) {
                dfs.delete(dstPath, true);
            }
        } catch (IOException ie) {
            logger.error(ie.getMessage());
            return false;
        }
        return true;
    }

    public boolean mkdir(String dst) {
        Path dstPath = new Path(dst);
        try {
            if (!dfs.exists(dstPath)) {
                dfs.mkdirs(dstPath, FsPermission.getDefault());
            }
        } catch (IOException ie) {
            logger.error(ie.getMessage());
            return false;
        }
        return true;
    }

    public boolean rename(String source, String target) {
        Path sourcePath = new Path(source);
        Path targetPath = new Path(target);
        try {
            if (dfs.exists(sourcePath)) {
                dfs.rename(sourcePath, targetPath);
            }
        } catch (IOException ie) {
            logger.error(ie.getMessage());
            return false;
        }
        return true;
    }

    public void close() {
        try {
            dfs.close();
        } catch (IOException e) {
            logger.error(e.getMessage());
        }
    }

    public String readFile2(String hdfsUrl, HttpServletResponse response) {
        FSDataInputStream fis = null;
        Path path = new Path(hdfsUrl);
        BufferedReader bis = null;
        String str = "";
        try {
            FileSystem fs = FileSystem.get(URI.create(hdfsUrl), conf);

            if (fs.exists(path)) {
                fis = fs.open(path);
                InputStreamReader isr = new InputStreamReader(fis, "utf-8");
                bis = new BufferedReader(isr);
                String line;
                while ((line = bis.readLine()) != null) {
                    str = str + line;
                }
            } else {
                throw new RRException(DataDevelopmentBizExceptionEnum.MODEL_NOT_FOUND_IN_HDFS.getMessage());
            }

        } catch (UnsupportedEncodingException ex) {
            ex.printStackTrace();
        } catch (IOException ex) {
            ex.printStackTrace();
        } catch (IllegalArgumentException e) {
            logger.error(e.getMessage());
        }
        return str;
    }

    public Long getTableOrDbSize(String dir) throws IOException {
        FileSystem fs = FileSystem.get(conf);
        Long tableSize ;
        //判断路径是否存在
        if (fs.exists(new Path(dir))) {
            FileStatus stats = fs.getFileStatus(new Path(dir));
            tableSize = fs.getContentSummary(stats.getPath()).getLength();
            fs.close();
            return tableSize;
        }

        return null;
    }


}

配置属性：

bdp-manager:
        namenodestr: bigdata-test-1:8020,bigdata-test-2:8020
        namespace: ns
        nameport: 8020