centos7+hadoop3.1.2完全分布式集群搭建

• hosts

127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
192.168.252.31 mini1
192.168.252.32 mini2
192.168.252.33 mini3

• 关闭防火墙

systemctl stop firewalld
systemctl disable firewalld

• 安装jdk

tar -zvxf jdk-8u111-linux-x64.tar.gz
export JAVA_HOME=/usr/local/jdk1.8.0_111
export PATH=$PATH:$JAVA_HOME/bin

• SSH免密登录

ssh-keygen
ssh-copy-id mini1
ssh-copy-id mini2
ssh-copy-id mini3

• 安装hadoop

tar zxvf hadoop-3.1.2.tar.gz -C /root/apps
export HADOOP_HOME=/root/apps/hadoop-3.1.2
export PATH=$PATH:$JAVA_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$HIVE_HOME/bin

• hadoop-env.sh

export JAVA_HOME=/usr/local/jdk1.8.0_111

• core-site.xml

<!-- 指定HDFS中NameNode的地址 -->
<property>
	<name>fs.defaultFS</name>
	<value>hdfs://mini1:9000</value>
</property>
<!-- 指定hadoop运行时产生文件的存储目录 -->
<property>
	 <name>hadoop.tmp.dir</name>
	<value>/root/data/tmp</value>
</property>

• hdfs-site.xml

<property>
	 <name>dfs.namenode.secondary.http-address</name>
	<value>mini1:50090</value>
</property>
<property>
	<name>dfs.namenode.name.dir</name>
 	<value>/root/dfs/name</value>
</property>
<property>
	<name>dfs.datanode.data.dir</name>
	<value>/root/dfs/data</value>
</property>
<property>
	<name>dfs.replication</name>
	<value>1</value>
</property>

• mapred-site.xml

<!-- 指定mr运行在yarn上 -->
<property>
	<name>mapreduce.framework.name</name>
 	<value>yarn</value>
</property>
<property>
	<name>mapreduce.jobhistory.address</name>
	<value>mini1:10020</value>
</property>
<property>
	<name>mapreduce.jobhistory.webapp.address</name>
	<value>mini1:19888</value>
</property>

• yarn-site.xml

<!-- reducer获取数据的方式 -->
<property>
	 <name>yarn.nodemanager.aux-services</name>
	 <value>mapreduce_shuffle</value>
</property>
<!-- 指定YARN的ResourceManager的地址 -->
<property>
	<name>yarn.resourcemanager.hostname</name>
	<value>mini1</value>
</property>
<property>
	<name>yarn.application.classpath</name>
	<value>使用【 hadoop classpath 】获取</value>
</property>

hadoop classpath

[root@mini1 ~]# /root/apps/hadoop-3.1.2/bin/hadoop classpath
/root/apps/hadoop-3.1.2/etc/hadoop:/root/apps/hadoop-3.1.2/share/hadoop/common/lib/*:/root/apps/hadoop-3.1.2/share/hadoop/common/*:/root/apps/hadoop-3.1.2/share/hadoop/hdfs:/root/apps/hadoop-3.1.2/share/hadoop/hdfs/lib/*:/root/apps/hadoop-3.1.2/share/hadoop/hdfs/*:/root/apps/hadoop-3.1.2/share/hadoop/mapreduce/lib/*:/root/apps/hadoop-3.1.2/share/hadoop/mapreduce/*:/root/apps/hadoop-3.1.2/share/hadoop/yarn:/root/apps/hadoop-3.1.2/share/hadoop/yarn/lib/*:/root/apps/hadoop-3.1.2/share/hadoop/yarn/*

• workers(也就是老版本的slaves)

mini2
mini3

给下面脚本行首加入如下配置

• start-dfs.sh
• stop-dfs.sh

HDFS_DATANODE_USER=root
HDFS_DATANODE_SECURE_USER=hdfs
HDFS_NAMENODE_USER=root
HDFS_SECONDARYNAMENODE_USER=root

• start-yarn.sh
• stop-yarn.sh

YARN_RESOURCEMANAGER_USER=root
HADOOP_SECURE_DN_USER=yarn
YARN_NODEMANAGER_USER=root

• 关闭hadoop安全模式

hadoop dfsadmin -safemode leave

• 格式化

hadoop namenode -format

• 启动

sbin/start-all.sh