hdfs文件上传下载--client模式--Kerberos认证

 准备工作：从hadoop集群找到core-site.xml,hdfs-site.xml,yarn-site.xml配置文件，放到项目对应资源配置目录下。这种方式属性配置全，自己配置容易遗漏配置，导致

一些无法预料的错误。

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;

public class FileSystemConnection {private static String KERBEROS_PRINC = "admin@TDH";
    private static String KERBEROS_PRINC_KEYTAB_PATH =
            "D:\\admin.keytab";
    public static final Logger logger = LoggerFactory.getLogger(FileSystemConnection.class);

    public static FileSystem getHdfsConnection() {
        try {
            Configuration conf = new Configuration();
            conf.addResource("core-site.xml");
　　　　　　  conf.addResource("hdfs-site.xml");
　　　　　　  conf.addResource("yarn-site.xml");

            // 没开kerberos，注释下面两行
            logger.info("kerberos principal: {} ,keytab path: {}", KERBEROS_PRINC, KERBEROS_PRINC_KEYTAB_PATH);
            UserGroupInformation.setConfiguration(conf);
            UserGroupInformation.loginUserFromKeytab(KERBEROS_PRINC, KERBEROS_PRINC_KEYTAB_PATH);
            FileSystem fs = FileSystem.get(conf);
            return fs;
        } catch (Exception e) {
            throw new RuntimeException("get hdfs connection failure", e);
        }
    }

    private static void downLoadFile() throws IOException {
　　　　　FileSystem dfs=getHdfsConnection();
        String src="/tmp/tdt";
        String dest="D:\\TestFs";
        Path path = new Path(src);
        try {
            if (dfs.exists(path)) {
                File file = new File(dest + File.separator + path.getName());
                file.createNewFile();
                InputStream in = dfs.open(path);
                OutputStream out = new FileOutputStream(file);
                IOUtils.copyBytes(in, out, conf);
                in.close();
                out.close();
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

 

转载于:https://www.cnblogs.com/jvStarBlog/p/11416096.html