Hadoop的安装部署---------＜版本3.1.1＞

概况

• 版本： 3.1.1
• 节点： 3个

[root@doitedu01 ~]# tar -zxf hadoop-3.1.1.tar.gz -C /opt/apps/              解压到指定目录

配置文件

hadoop-env.sh

export JAVA_HOME=/opt/apps/jdk1.8.0_191/

core-site.xml

<configuration>

<property>
<name>dfs.replication</name>
<value>2</value>
</property>

<property>
<name>fs.defaultFS</name>
<value>hdfs://doitedu01:8020/</value>
</property>

<property>
<name>hadoop.http.staticuser.user</name>
<value>root</value>
</property>

<property>
<name>dfs.permissions.enabled</name>
<value>false</value>
</property>

<property>
<name>hadoop.proxyuser.root.hosts</name>
<value>*</value>
</property>

<property>
<name>hadoop.proxyuser.root.groups</name>
<value>*</value>
</property>

<property>
<name>hadoop.proxyuser.root.users</name>
<value>hdfs,impala,hive,hbase,yarn</value>
</property>

</configuration>

hdfs-site.xml

<configuration>
<property>
<name>dfs.permissions.enabled</name>
<value>false</value>
</property>

<property>
<name>dfs.replication</name>
<value>2</value>
</property>

<property>
<name>dfs.datanode.data.dir</name>
<value>/opt/data/hdpdata/data</value>
</property>

<property>
<name>dfs.namenode.name.dir</name>
<value>/opt/data/hdpdata/name</value>
</property>

<property>
<name>dfs.namenode.checkpoint.dir</name>
<value>/opt/data/hdpdata/secname/</value>
</property>

<property>
<name>dfs.namenode.rpc-address</name>
<value>doitedu01:8020</value>
</property>

<property>
  <name>dfs.namenode.secondary.http-address</name>
  <value>doitedu02:50090</value>
</property>
</configuration>

yarn-site.xml

<configuration>
<property>
<name>yarn.resourcemanager.hostname</name>
<value>doitedu01</value>
</property>

<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>

<property>
<name>yarn.nodemanager.vmem-check-enabled</name>
<value>false</value>
</property>

<property>
<name>yarn.nodemanager.resource.memory-mb</name>
<value>4096</value>
</property>

<property>
<name>yarn.nodemanager.resource.cpu-vcores</name>
<value>4</value>
</property>
</configuration>

mapred-site.xml

<configuration>

<property>
    <name>mapreduce.framework.name</name>
    <value>yarn</value>
</property>

<property>
    <name>yarn.app.mapreduce.am.env</name>
    <value>HADOOP_MAPRED_HOME=/opt/apps/hadoop-3.1.1</value>
</property>

<property>
    <name>mapreduce.map.env</name>
    <value>HADOOP_MAPRED_HOME=/opt/apps/hadoop-3.1.1</value>
</property>

<property>
    <name>mapreduce.reduce.env</name>
    <value>HADOOP_MAPRED_HOME=/opt/apps/hadoop-3.1.1</value>
</property>

<!--
<property>
    <name>mapreduce.reduce.java.opts</name>
    <value>-Xmx3072m</value>
</property>
-->

<property>
    <name>mapreduce.jobhistory.address</name>
    <value>doitedu01:10020</value>
</property>

<property>
    <name>mapreduce.jobhistory.webapp.address</name>
    <value>doitedu01:19888</value>
</property>

</configuration>

workers

[root@doitedu01 hadoop]# vi workers 
doitedu01
doitedu02
doitedu03

启动脚本添加安全用户定义

# 在start-dfs.sh stop-dfs.sh 开头加上：
HDFS_DATANODE_USER=root
HDFS_DATANODE_SECURE_USER=hdfs
HDFS_NAMENODE_USER=root
HDFS_SECONDARYNAMENODE_USER=root 

# 在start-yarn.sh stop-yarn.sh 开头加上：
YARN_RESOURCEMANAGER_USER=root
HADOOP_SECURE_DN_USER=yarn
YARN_NODEMANAGER_USER=root

/etc/profile 环境变量

export JAVA_HOME=/opt/apps/jdk1.8.0_191/
export HADOOP_HOME=/opt/apps/hadoop-3.1.1
export PATH=$PATH:$JAVA_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin

分发安装包到别的节点

[root@doitedu01 apps]# scp -r hadoop-3.1.1/ doitedu02:$PWD
[root@doitedu01 apps]# scp -r hadoop-3.1.1/ doitedu03:$PWD

初始化，启动

hdfs namenode -format

start-dfs.sh

启动后，可以打开如下地址查看web控制台 http://doitedu01:9870/