hadoop集群配置

hadoop集群配置

---

	hadoop102	hadoop103	hadoop104
HDFS	NameNode、DataNode	DataNode	SecondaryNameNode、DataNode
YARN	NodeManager	ResourceManager、NodeManager	NodeManager

 1. NameNode和SecondaryNameNode不要安装在同一台服务器上
 2. ResourceManager不要和NameNode和SecondaryNameNode安装在同一台服务器上

配置相关

1) core-site.xml

<!-- [yunmoran@hadoop102 hadoop]$ vim core-site.xml -->
<configuration>
    <!-- 指定NameNode的地址 -->  
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://hadoop102:8020</value>
    </property>
    <!-- 指定hadoop数据的存储目录 -->
    <property>
        <name>hadoop.tmp.dir</name>
        <value>/opt/my_hadoop/hadoop-3.1.3/data</value>
    </property>
    <!-- 配置HDFS网页登录使用的静态用户为yunmoran -->
    <property>
        <name>hadoop.http.staticuser.user</name>
        <value>yunmoran</value>
    </property>
</configuration>

2) hdfs-site.xml

<!-- [yunmoran@hadoop102 hadoop]$ vim hdfs-site.xml  -->
<configuration>
	<!-- nn web端访问地址-->
	<property>
        <name>dfs.namenode.http-address</name>
        <value>hadoop102:9870</value>
    </property>
	<!-- 2nn web端访问地址-->
    <property>
        <name>dfs.namenode.secondary.http-address</name>
        <value>hadoop104:9868</value>
    </property>
</configuration>

3) yarn-site.xml

<!-- [yunmoran@hadoop102 hadoop]$ vim yarn-site.xml  -->
<configuration>
    <!-- 指定MR走shuffle -->
    <property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
    </property>
    <!-- 指定ResourceManager的地址-->
    <property>
        <name>yarn.resourcemanager.hostname</name>
        <value>hadoop103</value>
    </property>
    <!-- 环境变量的继承 -->
    <property>
        <name>yarn.nodemanager.env-whitelist</name>
        <value>
JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_MAPRED_HOME
        </value>
    </property>
    <!-- 开启日志聚集功能 -->
    <property>
        <name>yarn.log-aggregation-enable</name>
        <value>true</value>
    </property>
    <!-- 设置日志聚集服务器地址 -->
    <property>  
        <name>yarn.log.server.url</name>  
        <value>http://hadoop102:19888/jobhistory/logs</value>
    </property>
    <!-- 设置日志保留时间为7天 -->
    <property>
        <name>yarn.log-aggregation.retain-seconds</name>
        <value>604800</value>
    </property>
    <property>
        <name>yarn.application.classpath</name>
        <value>/opt/my_hadoop/hadoop-3.1.3/etc/hadoop:/opt/my_hadoop/hadoop-3.1.3/share/hadoop/common/lib/*:/opt/my_hadoop/hadoop-3.1.3/share/hadoop/common/*:/opt/my_hadoop/hadoop-3.1.3/share/hadoop/hdfs:/opt/my_hadoop/hadoop-3.1.3/share/hadoop/hdfs/lib/*:/opt/my_hadoop/hadoop-3.1.3/share/hadoop/hdfs/*:/opt/my_hadoop/hadoop-3.1.3/share/hadoop/mapreduce/lib/*:/opt/my_hadoop/hadoop-3.1.3/share/hadoop/mapreduce/*:/opt/my_hadoop/hadoop-3.1.3/share/hadoop/yarn:/opt/my_hadoop/hadoop-3.1.3/share/hadoop/yarn/lib/*:/opt/my_hadoop/hadoop-3.1.3/share/hadoop/yarn/*
        </value>
    </property>
</configuration>

4) mapred-site.xml

<!-- [yunmoran@hadoop102 hadoop]$ vim mapred-site.xml  -->
<configuration>
	<!-- 指定MapReduce程序运行在Yarn上 -->
    <property>
        <name>mapreduce.framework.name</name>
        <value>yarn</value>
    </property>
    <!-- 历史服务器端地址 -->
    <property>
        <name>mapreduce.jobhistory.address</name>
        <value>hadoop102:10020</value>
    </property>
    <!-- 历史服务器web端地址 -->
    <property>
        <name>mapreduce.jobhistory.webapp.address</name>
        <value>hadoop102:19888</value>
    </property>
    <!-- hadoop路径,不加可能导致后期运行出错,通过classpath获得数据 -->
    <!-- [yunmoran@hadoop102 hadoop]$ hadoop classpath -->
    <property>
        <name>mapreduce.application.classpath</name>
        <value>/opt/my_hadoop/hadoop-3.1.3/etc/hadoop:/opt/my_hadoop/hadoop-3.1.3/share/hadoop/common/lib/*:/opt/my_hadoop/hadoop-3.1.3/share/hadoop/common/*:/opt/my_hadoop/hadoop-3.1.3/share/hadoop/hdfs:/opt/my_hadoop/hadoop-3.1.3/share/hadoop/hdfs/lib/*:/opt/my_hadoop/hadoop-3.1.3/share/hadoop/hdfs/*:/opt/my_hadoop/hadoop-3.1.3/share/hadoop/mapreduce/lib/*:/opt/my_hadoop/hadoop-3.1.3/share/hadoop/mapreduce/*:/opt/my_hadoop/hadoop-3.1.3/share/hadoop/yarn:/opt/my_hadoop/hadoop-3.1.3/share/hadoop/yarn/lib/*:/opt/my_hadoop/hadoop-3.1.3/share/hadoop/yarn/*
        </value>
    </property>
</configuration>

5) hadoop-env.sh

# [yunmoran@hadoop102 hadoop]$ vim /opt/my_hadoop/hadoop-3.1.3/etc/hadoop/hadoop-env.sh

export JAVA_HOME=/opt/my_jdk/jdk1.8.0_212

6) 群起集群,配置workers

# 在/opt/my_hadoop/hadoop-3.1.3/etc/hadoop下的workers中添加
# [yunmoran@hadoop102 hadoop]$ vim /opt/my_hadoop/hadoop-3.1.3/etc/hadoop/workers 
hadoop102
hadoop103
hadoop104
# 注:该文件中添加的内容结尾不允许有空格,文件中不允许有空行

7) 同步配置文件到不同的服务器上

sh xsync.sh /opt/my_hadoop

# 如果传输失败,就是权限不够,给另外的虚拟机中的/opt足够的权限
sudo chmod 777 /opt

8) 格式化集群

# 这里yunmoran这个用户也需要设置ssh免密登录
# 集群第一次启动时需要格式化namenode结点,产生新的集群id
[yunmoran@hadoop102 hadoop-3.1.3]$ hdfs namenode -format

9) 启动HDFS、YARN、历史服务器

[yunmoran@hadoop102 hadoop-3.1.3]$ sbin/start-dfs.sh
[yunmoran@hadoop103 hadoop-3.1.3]$ sbin/start-yarn.sh
[yunmoran@hadoop102 ~]$ mapred --daemon start historyserver

10) 查看HDFS、YARN、历史服务器是否启动成功

http://hadoop102:9870
http://hadoop103:8088
http://hadoop102:19888/jobhistory

11) 测试

# 在服务器创建一个文件夹
[yunmoran@hadoop102 hadoop-3.1.3]$ hadoop fs -mkdir /input
# 查看HDFS中是否存在该目录