大数据平台的安全认证Kerberos是一个比较麻烦的东西,但往往为了平台的安全需要开启,开启后使用JavaAPI操作HDFS,需要进行用户登陆认证。
1. 首先需要开启CDH的kerberos,开启Kerberos可以参考_CDH配置kerberos_。
2.开启后,下载 krb5.conf、hdfs-site.xml(hdfs的配置文件)和认证用户的.keytab文件
3.下面贴出来测试使用的代码
import org.apache.commons.io.filefilter.FileFileFilter; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.log4j.Logger; /** * @Description: HDFS的操作类 * @author: WH * @date: 2016-8-8 下午7:13:11 */ public class HdfsService { // 日志 private static Logger log =Logger.getLogger(HdfsService.class); private static Configuration conf = null; public String defaultAddress = "webhdfs://10.134.161.108:50070/";//设置hdfs的连接方式为webhdfs,通过HTTP访问hdfs private HdfsService() { // TODO Auto-generated method stub conf = new Configuration(); System.setProperty("java.security.krb5.conf","C:/Users/user/Desktop/pwmx/krb5.conf");//设置kerberos配置信息 conf.set("fs.defaultFS", defaultAddress);//namenode地址 conf.set("fs.hdfs.impl",org.apache.hadoop.hdfs.DistributedFileSystem.class.getName()); conf.set("fs.file.impl",org.apache.hadoop.fs.LocalFileSystem.class.getName()); conf.set("fs.webhdfs.impl",org.apache.hadoop.hdfs.web.WebHdfsFileSystem.class.getName()); conf.setBoolean("hadoop.security.authentication",true); conf.set("hadoop.security.authentication","kerberos"); conf.set("dfs.namenode.kerberos.principal","hdfs/_HOST@TEST.COM");//hdfs-site.xml中配置信息 conf.set("dfs.datanode.kerberos.principal","hdfs/_HOST@TEST.COM");//hdfs-site.xml中配置信息 UserGroupInformation.setConfiguration(conf); try { UserGroupInformation.loginUserFromKeytab("YJ100001","C:/Users/user/Desktop/pwmx/YJ100001.keytab");//kerberos 认证 UserGroupInformation.getLoginUser(); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); } } /** * @param path * @return * @throws IOException */ public boolean exits(String path) throws Exception { FileSystem fs = FileSystem.get(conf); return fs.exists(new Path(path)); } public static void main(String[] args){ HdfsService hd = new HdfsService(); try { System.out.println(hd.exits("/")); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); } } }