【Apache】Hadoop 3.2.1集群集成Kerberos

系列文章传送门

前篇：【Apache】Hadoop 3.2.1集群搭建
中篇：Kerberos的安装与配置
后篇：【Apache】Hadoop 3.2.1集群集成Kerberos

参考资料

1、《Hadoop权威指南》
2、http://t.csdn.cn/Yrq0Q

1. 创建HDFS相关的Kerberos账户并导出keytab文件

# 在每个节点都创建存放keytab的文件夹
mkdir -p /etc/security/keytabs

1.1 master节点

kadmin   # 输入密码123456, 进入Kerberos的admin后台

# 防止启动或者操作的过程中需要输入密码，创建免密登录的keytab文件
addprinc -randkey hdfs/master@HADOOP.COM                           
ktadd -k /etc/security/keytabs/hdfs.keytab hdfs/master@HADOOP.COM 
quit

klist -ket /etc/security/keytabs/hdfs.keytab

cd /etc/security/keytabs/
cp hdfs.keytab /opt/bigdata/hadoop-3.2.1/etc/hadoop/

1.2 slave1节点

kadmin   # 输入密码123456, 进入Kerberos的admin后台

addprinc -randkey hdfs/slave1@HADOOP.COM
ktadd -k /etc/security/keytabs/hdfs.keytab hdfs/slave1@HADOOP.COM
quit

klist -ket /etc/security/keytabs/hdfs.keytab

cd /etc/security/keytabs/
cp hdfs.keytab /opt/bigdata/hadoop-3.2.1/etc/hadoop/

1.3 slave2节点

kadmin   # 输入密码123456, 进入Kerberos的admin后台

addprinc -randkey hdfs/slave2@HADOOP.COM
ktadd -k /etc/security/keytabs/hdfs.keytab hdfs/slave2@HADOOP.COM
quit

klist -ket /etc/security/keytabs/hdfs.keytab

cd /etc/security/keytabs/
cp hdfs.keytab /opt/bigdata/hadoop-3.2.1/etc/hadoop/

2. 修改HDFS配置文件

cd $HADOOP_HOME/etc/hadoop

2.1 yarn-env.sh

vim yarn-env.sh

export JAVA_HOME=/usr/local/jdk1.8.0_351 

xsync $HADOOP_HOME/etc/hadoop/yarn-env.sh

2.2 mapred-env.sh

vim mapred-env.sh

export JAVA_HOME=/usr/local/jdk1.8.0_351 

xsync $HADOOP_HOME/etc/hadoop/mapred-env.sh

2.3 core-site.xml

vim core-site.xml

<configuration>

  <property>
    <name>fs.defaultFS</name>
    <value>hdfs://master:9000</value>
  </property>

  <property>
    <name>hadoop.tmp.dir</name>
    <value>/opt/bigdata/hadoop-3.2.1/tmp</value>
  </property>

  <property>
    <name>io.file.buffer.size</name>
    <value>131072</value>
  </property>

  <property>
    <name>hadoop.security.authorization</name>
    <value>true</value>
    <description>是否开启hadoop的安全认证</description>
  </property>

  <property>
    <name>hadoop.security.authentication</name>
    <value>kerberos</value>
    <description>使用kerberos作为hadoop的安全认证方案</description>
  </property>

  <property>
    <name>hadoop.rpc.protection</name>
    <value>authentication</value>
  </property>

  <property>
    <name>hadoop.security.auth_to_local</name>
    <value>
        RULE:[2:$1@$0](hdfs@.*HADOOP.COM)s/.*/hdfs/
        RULE:[2:$1@$0](yarn@.*HADOOP.COM)s/.*/yarn/
        DEFAULT
    </value>
  </property>

  <property>
    <name>hadoop.proxyuser.root.hosts</name>
    <value>*</value>
  </property>

  <property>
    <name>hadoop.proxyuser.root.groups</name>
    <value>*</value>
  </property>

  <property>
    <name>hadoop.proxyuser.hdfs.hosts</name>
    <value>*</value>
  </property>

  <property>
    <name>hadoop.proxyuser.hdfs.groups</name>
    <value>*</value>
  </property>

  <property>
    <name>hadoop.proxyuser.yarn.hosts</name>
    <value>*</value>
  </property>

  <property>
    <name>hadoop.proxyuser.yarn.groups</name>
    <value>*</value>
  </property>

  <property>
      <name>hadoop.proxyuser.hive.hosts</name>
      <value>*</value>
  </property>

  <property>
      <name>hadoop.proxyuser.hive.groups</name>
      <value>*</value>
  </property>

</configuration>

xsync $HADOOP_HOME/etc/hadoop/core-site.xml

2.4 hdfs-site.xml

vim hdfs-site.xml

<configuration>

   <property>
        <name>dfs.namenode.name.dir</name>
        <value>/data/nn</value>
   </property>

   <property>
        <name>dfs.datanode.data.dir</name>
        <value>/data/dn</value>
   </property>

   <property>
        <name>dfs.namenode.http-address</name>
        <value>master:9870</value>
   </property>
   
    <property>
        <name>dfs.namenode.secondary.http-address</name>
        <value>slave2:9868</value>
   </property>
   
   <property>
        <name>dfs.namenode.secondary.https-address</name>
        <value>slave2:9869</value>
        <description>HTTPS web UI address for the Secondary NameNode.</description>
   </property>

   <property>
        <name>dfs.namenode.hosts</name>
        <value>master,slave2</value>
   </property>

   <property>
        <name>dfs.namenode.kerberos.principal</name>
        <value>hdfs/master@HADOOP.COM</value>
        <description>namenode对应的kerberos账户</description>
   </property>

   <property>
        <name>dfs.namenode.keytab.file</name>
        <value>/opt/bigdata/hadoop-3.2.1/etc/hadoop/hdfs.keytab</value>
        <description>指定namenode需要用的keytab文件在哪里</description>
   </property>

   <property>
        <name>dfs.namenode.kerberos.internal.spnego.principal</name>
        <value>hdfs/_HOST@HADOOP.COM</value>
        <description>https 相关（如开启namenodeUI）使用的账户</description>
   </property>

   <property>
        <name>dfs.secondary.namenode.kerberos.principal</name>
        <value>hdfs/_HOST@HADOOP.COM</value>
        <description>secondarynamenode使用的账户</description>
   </property>

   <property>
        <name>dfs.secondary.namenode.keytab.file</name>
        <value>/opt/bigdata/hadoop-3.2.1/etc/hadoop/hdfs.keytab</value>
        <description>sn对应的keytab文件</description>
   </property>

   <property>
        <name>dfs.secondary.namenode.kerberos.internal.spnego.principal</name>
        <value>hdfs/_HOST@HADOOP.COM</value>
        <description>sn需要开启http页面用到的账户</description>
   </property>

   <property>
        <name>dfs.datanode.data.dir.perm</name>
        <value>700</value>
   </property>

   <property>
        <name>dfs.datanode.address</name>
        <value>0.0.0.0:1004</value>
   </property>

   <property>
        <name>dfs.datanode.http.address</name>
        <value>0.0.0.0:1006</value>
   </property>

   <property>
        <name>dfs.datanode.https.address</name>
        <value>0.0.0.0:9865</value>
        <description>HTTPS web UI address for the Data Node.</description>
   </property>

   <property>
        <name>dfs.datanode.kerberos.principal</name>
        <value>hdfs/_HOST@HADOOP.COM</value>
        <description>datanode用到的账户</description>
   </property>

   <property>
        <name>dfs.datanode.keytab.file</name>
        <value>/opt/bigdata/hadoop-3.2.1/etc/hadoop/hdfs.keytab</value>
        <description>datanode用到的keytab文件路径</description>
   </property>

   <property>
        <name>dfs.block.access.token.enable</name>
        <value>true</value>
   </property>

   <property>
        <name>dfs.permissions.supergroup</name>
        <value>hadoop</value>
   </property>

   <property>
        <name>dfs.replication</name>
        <value>3</value>
   </property>

   <property>
        <name>dfs.data.transfer.protection</name>
        <value>integrity</value>
   </property>

   <property>
        <name>dfs.encrypt.data.transfer</name>
        <value>true</value>
        <description>数据传输协议激活数据加密</description>
   </property>
   
   <property>
        <name>dfs.web.authentication.kerberos.principal</name>
        <value>hdfs/_HOST@HADOOP.COM</value>
        <description>web hdfs 使用的账户</description>
   </property>

   <property>
        <name>dfs.web.authentication.kerberos.keytab</name>
        <value>/opt/bigdata/hadoop-3.2.1/etc/hadoop/hdfs.keytab</value>
        <description>对应的keytab文件</description>
   </property>

   <property>
        <name>dfs.http.policy</name>
        <value>HTTPS_ONLY</value>
        <description>所有开启的web页面均使用https, 细节在ssl server 和client那个配置文件内配置</description>
   </property>

</configuration>

xsync $HADOOP_HOME/etc/hadoop/hdfs-site.xml

2.5 yarn-site.xml

vim yarn-site.xml

<configuration>

  <property>
     <name>yarn.nodemanager.aux-services</name>
     <value>mapreduce_shuffle</value>
  </property>

  <property>
     <name>yarn.resourcemanager.hostname</name>
     <value>slave1</value>
  </property>

  <property>
     <name>yarn.nodemanager.env-whitelist</name>
     <value>JAVA_HOME,HADOOP_HOME</value>
  </property>

 <!-- Resource Manager 服务的Kerberos主体 -->
 <property>
	 <name>yarn.resourcemanager.principal</name>
	 <value>hdfs/_HOST@HADOOP.COM</value>
 </property>

<!-- Resource Manager 服务的Kerberos密钥文件 -->
 <property>
     <name>yarn.resourcemanager.keytab</name>
     <value>/opt/bigdata/hadoop-3.2.1/etc/hadoop/hdfs.keytab</value>
 </property>

<!-- Node Manager 服务的Kerberos主体 -->
 <property>
     <name>yarn.nodemanager.principal</name>
     <value>hdfs/_HOST@HADOOP.COM</value>
 </property>

<!-- Node Manager 服务的Kerberos密钥文件 -->
 <property>
     <name>yarn.nodemanager.keytab</name>
     <value>/opt/bigdata/hadoop-3.2.1/etc/hadoop/hdfs.keytab</value>
 </property>
</configuration>

xsync yarn-site.xml

2.6 mapred-site.xml

vim mapred-site.xml

<configuration>

	<property>
		<name>mapreduce.framework.name</name>
		<value>yarn</value>
	</property>

	<!-- 历史服务器的Kerberos主体 -->
	<property>
		<name>mapreduce.jobhistory.keytab</name>
		<value>/opt/bigdata/hadoop-3.2.1/etc/hadoop/hdfs.keytab</value>
	</property>

	<!-- 历史服务器的Kerberos密钥文件 -->
	<property>
		<name>mapreduce.jobhistory.principal</name>
		<value>hdfs/_HOST@HADOOP.COM</value>
	</property>
	
</configuration>

xsync mapred-site.xml

3.创建HTTPS证书

# master节点执行
mkdir -p /opt/security/kerberos_https
cd /opt/security/kerberos_https

# 密码123456
openssl req -new -x509 -keyout bd_ca_key -out bd_ca_cert -days 9999 -subj '/C=CN/ST=beijing/L=beijing/O=test/OU=test/CN=test'

# 同步到所有节点
xsync /opt/security/kerberos_https

# 以下步骤每个节点都执行
cd /opt/security/kerberos_https

# 输入密码和确认密码：123456，此命令成功后输出keystore文件
keytool -keystore keystore -alias localhost -validity 9999 -genkey -keyalg RSA -keysize 2048 -dname "CN=test, OU=test, O=test, L=beijing, ST=beijing, C=CN"

# 输入密码和确认密码：123456，提示是否信任证书：输入yes，此命令成功后输出truststore文件
keytool -keystore truststore -alias CARoot -import -file bd_ca_cert

# 输入密码和确认密码：123456，此命令成功后输出cert文件
keytool -certreq -alias localhost -keystore keystore -file cert

# 此命令成功后输出cert_signed文件
openssl x509 -req -CA bd_ca_cert -CAkey bd_ca_key -in cert -out cert_signed -days 9999 -CAcreateserial -passin pass:123456

# 输入密码和确认密码：123456，是否信任证书，输入yes，此命令成功后更新keystore文件
keytool -keystore keystore -alias CARoot -import -file bd_ca_cert

# 输入密码和确认密码：123456
keytool -keystore keystore -alias localhost -import -file cert_signed

4. 编辑ssl-server.xml

cd ${HADOOP_HOME}/etc/hadoop
cp ssl-server.xml.example ssl-server.xml

# 在${HADOOP_HOME}/etc/hadoop目录构建ssl-server.xml文件
# 以下为ssl-server.xml的配置内容

<configuration>

    <property>
        <name>ssl.server.truststore.location</name>
        <value>/opt/security/kerberos_https/truststore</value>
    </property>

    <property>
        <name>ssl.server.truststore.password</name>
        <value>123456</value>
    </property>

    <property>
        <name>ssl.server.truststore.type</name>
        <value>jks</value>
    </property>

    <property>
        <name>ssl.server.truststore.reload.interval</name>
        <value>10000</value>
    </property>

    <property>
        <name>ssl.server.keystore.location</name>
        <value>/opt/security/kerberos_https/keystore</value>
    </property>

    <property>
        <name>ssl.server.keystore.password</name>
        <value>123456</value>
    </property>

    <property>
        <name>ssl.server.keystore.keypassword</name>
        <value>123456</value>
    </property>

    <property>
        <name>ssl.server.keystore.type</name>
        <value>jks</value>
    </property>

    <property>
        <name>ssl.server.exclude.cipher.list</name>
        <value>TLS_ECDHE_RSA_WITH_RC4_128_SHA,SSL_DHE_RSA_EXPORT_WITH_DES40_CBC_SHA,
        SSL_RSA_WITH_DES_CBC_SHA,SSL_DHE_RSA_WITH_DES_CBC_SHA,
        SSL_RSA_EXPORT_WITH_RC4_40_MD5,SSL_RSA_EXPORT_WITH_DES40_CBC_SHA,
        SSL_RSA_WITH_RC4_128_MD5</value>
    </property>
   
</configuration>

5. 编辑ssl-client.xml

cd ${HADOOP_HOME}/etc/hadoop
cp ssl-client.xml.example ssl-client.xml

# 在${HADOOP_HOME}/etc/hadoop目录构建ssl-client.xml文件

<configuration>

    <property>
        <name>ssl.client.truststore.location</name>
        <value>/opt/security/kerberos_https/truststore</value>
    </property>

    <property>
        <name>ssl.client.truststore.password</name>
        <value>123456</value>
    </property>

    <property>
        <name>ssl.client.truststore.type</name>
        <value>jks</value>
    </property>

    <property>
        <name>ssl.client.truststore.reload.interval</name>
        <value>10000</value>
    </property>

    <property>
        <name>ssl.client.keystore.location</name>
        <value>/opt/security/kerberos_https/keystore</value>
    </property>

    <property>
        <name>ssl.client.keystore.password</name>
        <value>123456</value>
    </property>

    <property>
        <name>ssl.client.keystore.keypassword</name>
        <value>123456</value>
    </property>

    <property>
        <name>ssl.client.keystore.type</name>
        <value>jks</value>
    </property>
    
</configuration>

6.开启HTTPS

# 修改hdfs-site.xml
vim  ${HADOOP_HOME}/etc/hadoop/hdfs-site.xml

	<property>
		<name>dfs.http.policy</name>
		<value>HTTPS_ONLY</value>
	</property>

xsync ${HADOOP_HOME}/etc/hadoop/hdfs-site.xml

7.启动集群

# master 节点
kinit -kt /etc/security/keytabs/hdfs.keytab hdfs/master@HADOOP.COM

# slave1 节点
kinit -kt /etc/security/keytabs/hdfs.keytab hdfs/slave1@HADOOP.COM

# slave2 节点
kinit -kt /etc/security/keytabs/hdfs.keytab hdfs/slave2@HADOOP.COM

# master 节点 root用户启动
cd ${HADOOP_HOME}/sbin
start-dfs.sh

# slave1 节点
cd ${HADOOP_HOME}/sbin
start-yarn.sh

如果启动集群时报错，可以查看$HADOOP_HOME/logs目录下的日志解决错误。