HDFS上配置kerberos（十一）

一、创建认证规则
二、创建Keytab文件
三、部署Kerberos Keytab文件
四、修改HDFS配置文件，包括
1）core-site.xml
2）hdfs-site.xml
五、启动namenode
六、启动datanode

步骤实施
1、创建认证规则
 [root@cdh1 training]# kadmin.local -q "addprinc -randkey hdfs/cdh1@ZGP.COM"
Authenticating as principal root/admin@ZGP.COM with password.
WARNING: no policy specified for hdfs/cdh1@ZGP.COM; defaulting to no policy
Principal "hdfs/cdh1@ZGP.COM" created.
[root@cdh1 training]#
[root@cdh1 training]# kadmin.local -q "addprinc -randkey hdfs/cdh2@ZGP.COM"
[root@cdh1 training]# kadmin.local -q "addprinc -randkey hdfs/cdh3@ZGP.COM"
[root@cdh1 training]# kadmin.local -q "addprinc -randkey HTTP/cdh1@ZGP.COM"
[root@cdh1 training]# kadmin.local -q "addprinc -randkey HTTP/cdh2@ZGP.COM"
[root@cdh1 training]# kadmin.local -q "addprinc -randkey HTTP/cdh3@ZGP.COM"

[root@cdh1 training]# kadmin
Authenticating as principal root/admin@ZGP.COM with password.
Password for root/admin@ZGP.COM: 
kadmin:  list_principals
HTTP/cdh1@ZGP.COM
HTTP/cdh2@ZGP.COM
HTTP/cdh3@ZGP.COM
K/M@ZGP.COM
hdfs/cdh1@ZGP.COM
hdfs/cdh2@ZGP.COM
hdfs/cdh3@ZGP.COM
kadmin/admin@ZGP.COM
kadmin/cdh1@ZGP.COM
kadmin/changepw@ZGP.COM
krbtgt/ZGP.COM@ZGP.COM
root/admin@ZGP.COM
kadmin:  


2、创建keytab文件

[root@cdh1 training]# cd /var/kerberos/krb5kdc
[root@cdh1 krb5kdc]# kadmin.local -q "xst -k hdfs-unmerged.keytab hdfs/cdh1@ZGP.COM"
Authenticating as principal root/admin@ZGP.COM with password.
Entry for principal hdfs/cdh1@ZGP.COM with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:hdfs-unmerged.keytab.
Entry for principal hdfs/cdh1@ZGP.COM with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:hdfs-unmerged.keytab.
Entry for principal hdfs/cdh1@ZGP.COM with kvno 2, encryption type des3-cbc-sha1 added to keytab WRFILE:hdfs-unmerged.keytab.
Entry for principal hdfs/cdh1@ZGP.COM with kvno 2, encryption type arcfour-hmac added to keytab WRFILE:hdfs-unmerged.keytab.
Entry for principal hdfs/cdh1@ZGP.COM with kvno 2, encryption type des-hmac-sha1 added to keytab WRFILE:hdfs-unmerged.keytab.
Entry for principal hdfs/cdh1@ZGP.COM with kvno 2, encryption type des-cbc-md5 added to keytab WRFILE:hdfs-unmerged.keytab.
[root@cdh1 krb5kdc]#
[root@cdh1 krb5kdc]# kadmin.local -q "xst -k hdfs-unmerged.keytab hdfs/cdh2@ZGP.COM"
[root@cdh1 krb5kdc]# kadmin.local -q "xst -k hdfs-unmerged.keytab hdfs/cdh3@ZGP.COM"


[root@cdh1 krb5kdc]# klist -ket hdfs-unmerged.keytab
Keytab name: WRFILE:hdfs-unmerged.keytab
KVNO Timestamp         Principal
---- ----------------- --------------------------------------------------------
   2 04/01/17 03:42:28 hdfs/cdh1@ZGP.COM (des3-cbc-sha1) 
   2 04/01/17 03:42:28 hdfs/cdh1@ZGP.COM (arcfour-hmac) 
   2 04/01/17 03:42:28 hdfs/cdh1@ZGP.COM (des-hmac-sha1) 
   2 04/01/17 03:42:28 hdfs/cdh1@ZGP.COM (des-cbc-md5) 
   2 04/01/17 03:42:33 hdfs/cdh2@ZGP.COM (des3-cbc-sha1) 
   2 04/01/17 03:42:33 hdfs/cdh2@ZGP.COM (arcfour-hmac) 
   2 04/01/17 03:42:33 hdfs/cdh2@ZGP.COM (des-hmac-sha1) 
   2 04/01/17 03:42:33 hdfs/cdh2@ZGP.COM (des-cbc-md5) 
   2 04/01/17 03:42:37 hdfs/cdh3@ZGP.COM (des3-cbc-sha1) 
   2 04/01/17 03:42:37 hdfs/cdh3@ZGP.COM (arcfour-hmac) 
   2 04/01/17 03:42:38 hdfs/cdh3@ZGP.COM (des-hmac-sha1) 
   2 04/01/17 03:42:38 hdfs/cdh3@ZGP.COM (des-cbc-md5)
[root@cdh1 krb5kdc]#

[root@cdh1 krb5kdc]# kadmin.local -q "xst -k HTTP.keytab HTTP/cdh1@ZGP.COM"
[root@cdh1 krb5kdc]# kadmin.local -q "xst -k HTTP.keytab HTTP/cdh2@ZGP.COM"
[root@cdh1 krb5kdc]# kadmin.local -q "xst -k HTTP.keytab HTTP/cdh3@ZGP.COM"

[root@cdh1 krb5kdc]# klist -ket HTTP.keytab
Keytab name: WRFILE:HTTP.keytab
KVNO Timestamp         Principal
---- ----------------- --------------------------------------------------------
   2 04/01/17 03:43:15 HTTP/cdh1@ZGP.COM (des3-cbc-sha1) 
   2 04/01/17 03:43:15 HTTP/cdh1@ZGP.COM (arcfour-hmac) 
   2 04/01/17 03:43:15 HTTP/cdh1@ZGP.COM (des-hmac-sha1) 
   2 04/01/17 03:43:15 HTTP/cdh1@ZGP.COM (des-cbc-md5) 
   2 04/01/17 03:43:18 HTTP/cdh2@ZGP.COM (des3-cbc-sha1) 
   2 04/01/17 03:43:19 HTTP/cdh2@ZGP.COM (arcfour-hmac) 
   2 04/01/17 03:43:19 HTTP/cdh2@ZGP.COM (des-hmac-sha1) 
   2 04/01/17 03:43:19 HTTP/cdh2@ZGP.COM (des-cbc-md5) 
   2 04/01/17 03:43:22 HTTP/cdh3@ZGP.COM (des3-cbc-sha1) 
   2 04/01/17 03:43:22 HTTP/cdh3@ZGP.COM (arcfour-hmac) 
   2 04/01/17 03:43:22 HTTP/cdh3@ZGP.COM (des-hmac-sha1) 
   2 04/01/17 03:43:22 HTTP/cdh3@ZGP.COM (des-cbc-md5)
[root@cdh1 krb5kdc]#
合并keytab文件
[root@cdh1 krb5kdc]# ktutil
ktutil:  rkt hdfs-unmerged.keytab
ktutil:  rkt HTTP.keytab
ktutil:  wkt hdfs.keytab
ktutil:  q
[root@cdh1 krb5kdc]#
验证
[root@cdh1 krb5kdc]# kinit -k -t hdfs.keytab hdfs/cdh1@ZGP.COM
[root@cdh1 krb5kdc]# kinit -k -t hdfs.keytab HTTP/cdh1@ZGP.COM
[root@cdh1 krb5kdc]#
缓存的信息
[root@cdh1 krb5kdc]# klist
Ticket cache: FILE:/tmp/krb5cc_0
Default principal: HTTP/cdh1@ZGP.COM


Valid starting     Expires            Service principal
03/31/17 03:28:19  04/01/17 03:28:19  krbtgt/ZGP.COM@ZGP.COM
        renew until 04/07/17 03:28:19
[root@cdh1 krb5kdc]#


3、部署Kerberos Keytab文件
[root@cdh1 krb5kdc]# cp hdfs.keytab /etc/hadoop/conf
[root@cdh1 krb5kdc]# scp hdfs.keytab cdh2:/etc/hadoop/conf
hdfs.keytab                                                                               100% 2066     2.0KB/s   00:00    
[root@cdh1 krb5kdc]# scp hdfs.keytab cdh3:/etc/hadoop/conf
hdfs.keytab                                                                               100% 2066     2.0KB/s   00:00    
[root@cdh1 krb5kdc]#
权限与用户组修改
[root@cdh1 krb5kdc]# cd /etc/hadoop/conf
[root@cdh1 conf]# ll | grep keytab
-rw------- 1 root root  2066 Mar 31 03:32 hdfs.keytab
[root@cdh1 conf]# chown hdfs:hadoop hdfs.keytab 
[root@cdh1 conf]# chmod 400 hdfs.keytab 
[root@cdh1 conf]# ll | grep keytab
-r-------- 1 hdfs hadoop  2066 Mar 31 03:32 hdfs.keytab
[root@cdh1 krb5kdc]# ssh cdh2 chown hdfs:hadoop /etc/hadoop/conf/hdfs.keytab
[root@cdh1 krb5kdc]# ssh cdh2 chmod 400 /etc/hadoop/conf/hdfs.keytab
[root@cdh1 krb5kdc]# ssh cdh3 chown hdfs:hadoop /etc/hadoop/conf/hdfs.keytab
[root@cdh1 krb5kdc]# ssh cdh3 chmod 400 /etc/hadoop/conf/hdfs.keytab
[root@cdh1 ~]# slaves.sh ls -l /etc/hadoop/conf/hdfs.keytab
cdh3: -r-------- 1 hdfs hadoop 2066 Mar 31 03:32 /etc/hadoop/conf/hdfs.keytab
cdh2: -r-------- 1 hdfs hadoop 2066 Mar 31 03:32 /etc/hadoop/conf/hdfs.keytab
[root@cdh1 ~]#


4、修改HDFS配置文件，包括
在修改配置文件前，先停掉所有hadoop服务

[root@cdh1 ~]# for S in `cd /etc/init.d;ls hadoop*`; do echo $S; done
hadoop-hdfs-namenode
hadoop-hdfs-secondarynamenode
hadoop-httpfs
hadoop-mapreduce-historyserver
hadoop-yarn-proxyserver
hadoop-yarn-resourcemanager
[root@cdh1 ~]#
关闭服务时请注意执行顺序
[root@cdh2 conf]# for S in `cd /etc/init.d;ls hadoop*`; do service $S stop; done
[root@cdh3 conf]# for S in `cd /etc/init.d;ls hadoop*`; do service $S stop; done
[root@cdh1 ~]# for S in `cd /etc/init.d;ls hadoop*`; do service $S stop; done


修改core-site.xml
[root@cdh1 conf]# vi core-site.xml
<property>
  <name>hadoop.security.authentication</name>
  <value>kerberos</value>
 </property>

 <property>
  <name>hadoop.security.authorization</name>
  <value>true</value>
 </property>

修改hdfs-site.xml

<!--kerberos security-->
 <property>
  <name>dfs.block.access.token.enable</name>
  <value>true</value>
 </property>

 <property>
  <name>dfs.datanode.data.dir.perm</name>
  <value>700</value>
 </property>

 <property>
  <name>dfs.namenode.keytab.file</name>
  <value>/etc/hadoop/conf/hdfs.keytab</value>
 </property>

 <property>
  <name>dfs.namenode.kerberos.principal</name>
  <value>hdfs/_HOST@ZGP.COM</value>
 </property>

 <property>
  <name>dfs.namenode.kerberos.https.principal</name>
  <value>HTTP/_HOST@ZGP.COM</value>
 </property>

 <property>
  <name>dfs.datanode.keytab.file</name>
  <value>/etc/hadoop/conf/hdfs.keytab</value>
 </property>

 <property>
  <name>dfs.datanode.kerberos.principal</name>
  <value>hdfs/_HOST@ZGP.COM</value>
 </property>

 <property>
  <name>dfs.datanode.kerberos.https.principal</name>
  <value>HTTP/_HOST@ZGP.COM</value>
 </property>

 <property>
  <name>dfs.datanode.address</name>
  <value>0.0.0.0:1004</value>
 </property>

 <property>
  <name>dfs.datanode.http.address</name>
  <value>0.0.0.0:1006</value>
 </property>

<!--webHDFS security-->

 <property>
  <name>dfs.webhdfs.enabled</name>
  <value>true</value>
 </property>

 <property>
  <name>dfs.web.authentication.kerberos.keytab</name>
  <value>/etc/hadoop/conf/hdfs.keytab</value>
 </property>

 <property>
  <name>dfs.web.authentication.kerberos.principal</name>
  <value>HTTP/_HOST@ZGP.COM</value>
 </property>

5、启动namenode
确保缓存没有ticket
[ root@cdh1 conf]# klist
Ticket cache: FILE:/tmp/krb5cc_0
Default principal: HTTP/cdh1@ZGP.COM


Valid starting     Expires            Service principal
03/31/17 03:28:19  04/01/17 03:28:19  krbtgt/ZGP.COM@ZGP.COM
        renew until 04/07/17 03:28:19
[root@cdh1 conf]# kdestroy 
[root@cdh1 conf]# klist
klist: No credentials cache found (ticket cache FILE:/tmp/krb5cc_0)
[root@cdh1 conf]#
[root@cdh1 conf]# kinit -k -t /etc/hadoop/conf/hdfs.keytab hdfs/cdh1@ZGP.COM  <==获取tgt
[root@cdh1 conf]# /etc/init.d/hadoop-hdfs-namenode start
Starting Hadoop namenode:[  OK  ]
starting namenode, logging to /var/log/hadoop-hdfs/hadoop-hdfs-namenode-cdh1.out
[root@cdh1 conf]# jps
4301 NameNode
4347 Jps
[root@cdh1 conf]# hdfs dfs -ls /
Found 3 items
drwxrwxrwt   - hdfs hadoop          0 2017-03-29 22:40 /tmp
drwx------   - hdfs hadoop          0 2017-03-29 22:44 /user
drwxr-xr-x   - hdfs hadoop          0 2017-03-29 04:24 /yarn
[root@cdh1 conf]#
[root@cdh1 conf]# ssh cdh2 kinit -kt /etc/hadoop/conf/hdfs.keytab hdfs/cdh2@ZGP.COM
[root@cdh1 conf]# ssh cdh3 kinit -kt /etc/hadoop/conf/hdfs.keytab hdfs/cdh3@ZGP.COM
[root@cdh1 conf]#
[root@cdh2 training]# /etc/init.d/hadoop-hdfs-datanode start
[root@cdh3 training]# /etc/init.d/hadoop-hdfs-datanode start
此时datanode异常报如下错误
2017-04-01 04:05:44,790 WARN org.apache.hadoop.hdfs.server.datanode.DataNode: Problem connecting to server: cdh1/192.168.123.20:8020
解决方法：
[root@cdh2 default]# cd /etc/default/
[root@cdh2 default]# vi hadoop-hdfs-datanode
export HADOOP_SECURE_DN_USER=hdfs
export HADOOP_SECURE_DN_PID_DIR=/var/run/hadoop-hdfs
export HADOOP_SECURE_DN_LOG_DIR=/var/log/hadoop-hdfs
export JSVC_HOME=/usr/lib/bigtop-utils
~

[root@cdh2 default]# scp hadoop-hdfs-datanode cdh3:/etc/default/

重新启动datanode服务即可