Hadoop分布式集群

CentOS7系统基础配置

1. ip

2. hostname

3. hosts

4. firewalld

5. crontab-ntpdate

6. JDK（/data/soft/jdk8）

7. Hadoop（/data/soft/hadoop3）

---

Hadoop分布式集群配置文件

1. /data/soft/hadoop3/etc/hadoop/hadoop-env.sh

export JAVA_HOME=/data/soft/jdk8
export HADOOP_LOG_DIR=/data/hadoop_repo/logs/hadoop

2. /data/soft/hadoop3/etc/hadoop/core-site.xml

<configuration>
	<property>
		<name>fs.defaultFS</name>
		<value>hdfs://bigdata01:9000</value>
	</property>
	<property>
		<name>hadoop.tmp.dir</name>
		<value>/data/hadoop_repo</value>
	</property>
	<property>
		<name>fs.trash.interval</name>
		<value>0</value>
	</property>
</configuration>

3. /data/soft/hadoop3/etc/hadoop/hdfs-site.xml

<configuration>
	<property>
		<name>dfs.replication</name>
		<value>2</value>
	</property>
</configuration>

4. /data/soft/hadoop3/etc/hadoop/mapred-site.xml

<configuration>
	<property>
		<name>mapreduce.framework.name</name>
		<value>yarn</value>
	</property>
</configuration>

5. /data/soft/hadoop3/etc/hadoop/yarn-site.xml

<configuration>
	<property>
		<name>yarn.nodemanager.aux-services</name>
		<value>mapreduce_shuffle</value>
	</property>
	<property>
		<name>yarn.nodemanager.env-whitelist</name>
		<value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_MAPRED_HOME</value>
	</property>
</configuration>

6. /data/soft/hadoop3/etc/hadoop/workers

bigdata02
bigdata03

7. /data/soft/hadoop3/sbin/start-dfs.sh

HDFS_DATANODE_USER=root
HDFS_DATANODE_SECURE_USER=hdfs
HDFS_NAMENODE_USER=root
HDFS_SECONDARYNAMENODE_USER=root

8. /data/soft/hadoop3/sbin/stop-dfs.sh

HDFS_DATANODE_USER=root
HDFS_DATANODE_SECURE_USER=hdfs
HDFS_NAMENODE_USER=root
HDFS_SECONDARYNAMENODE_USER=root

9. /data/soft/hadoop3/sbin/start-yarn.sh

YARN_RESOURCEMANAGER_USER=root
HADOOP_SECURE_DN_USER=yarn
YARN_NODEMANAGER_USER=root

10. /data/soft/hadoop3/sbin/stop-yarn.sh

YARN_RESOURCEMANAGER_USER=root
HADOOP_SECURE_DN_USER=yarn
YARN_NODEMANAGER_USER=root

---

完成上述配置后需要分别在每台服务器上配置ssh免密登录