电商数仓3.0 数据采集模块之Hadoop安装

集群所有进程查看脚本

在/home/scorpion/bin目录下创建脚本xcall

[scorpion@warehouse102 bin]$ vim xcall
#! /bin/bash
#集群所有进程查看脚本

for i in warehouse102 warehouse103 warehouse104
do
    echo --------- $i ----------
    ssh $i "$*"
done

[scorpion@warehouse102 bin]$ chmod 777 xcall

Hadoop安装

参考文档：https://blog.csdn.net/qq_40180229/article/details/100118521
网盘链接：https://pan.baidu.com/s/13PtZPMvmRLXn243hS1X-pQ
提取码：v6t6

集群规划

配置core-site.xml

[scorpion@warehouse102 hadoop]$ vim $HADOOP_HOME/etc/hadoop/core-site.xml

<configuration>
	<!-- 指定NameNode的地址 -->
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://warehouse102:9820</value>
</property>
<!-- 指定hadoop数据的存储目录 -->
    <property>
        <name>hadoop.tmp.dir</name>
        <value>/opt/module/hadoop-3.1.3/data</value>
</property>

<!-- 配置HDFS网页登录使用的静态用户为scorpion -->
    <property>
        <name>hadoop.http.staticuser.user</name>
        <value>scorpion</value>
</property>

<!-- 配置该scorpion(superUser)允许通过代理访问的主机节点 -->
    <property>
        <name>hadoop.proxyuser.scorpion.hosts</name>
        <value>*</value>
</property>
<!-- 配置该scorpion(superUser)允许通过代理用户所属组 -->
    <property>
        <name>hadoop.proxyuser.scorpion.groups</name>
        <value>*</value>
</property>
<!-- 配置该scorpion(superUser)允许通过代理的用户-->
    <property>
        <name>hadoop.proxyuser.scorpion.groups</name>
        <value>*</value>
</property>
</configuration>

配置hdfs-site.xml

[scorpion@warehouse102 hadoop]$ vim $HADOOP_HOME/etc/hadoop/hdfs-site.xml

<configuration>
	<!-- nn web端访问地址-->
	<property>
        <name>dfs.namenode.http-address</name>
        <value>warehouse102:9870</value>
    </property>
    
	<!-- 2nn web端访问地址-->
    <property>
        <name>dfs.namenode.secondary.http-address</name>
        <value>warehouse104:9868</value>
    </property>
    
    <!-- 测试环境指定HDFS副本的数量1 -->
    <property>
        <name>dfs.replication</name>
        <value>1</value>
    </property>
</configuration>

配置yarn-site.xml

[scorpion@warehouse102 hadoop]$ vim $HADOOP_HOME/etc/hadoop/yarn-site.xml

<configuration>
	<!-- 指定MR走shuffle -->
    <property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
    </property>
    
    <!-- 指定ResourceManager的地址-->
    <property>
        <name>yarn.resourcemanager.hostname</name>
        <value>wawrehouse103</value>
    </property>
    
    <!-- 环境变量的继承 -->
    <property>
        <name>yarn.nodemanager.env-whitelist</name>
        <value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_MAPRED_HOME</value>
    </property>
    
    <!-- yarn容器允许分配的最大最小内存 -->
    <property>
        <name>yarn.scheduler.minimum-allocation-mb</name>
        <value>512</value>
    </property>
    <property>
        <name>yarn.scheduler.maximum-allocation-mb</name>
        <value>4096</value>
    </property>
    
    <!-- yarn容器允许管理的物理内存大小 -->
    <property>
        <name>yarn.nodemanager.resource.memory-mb</name>
        <value>4096</value>
    </property>
    
    <!-- 关闭yarn对物理内存和虚拟内存的限制检查 -->
    <property>
        <name>yarn.nodemanager.pmem-check-enabled</name>
        <value>false</value>
    </property>
    <property>
        <name>yarn.nodemanager.vmem-check-enabled</name>
        <value>false</value>
    </property>
    <!-- 开启日志聚集功能 -->
<property>
    <name>yarn.log-aggregation-enable</name>
    <value>true</value>
</property>

<!-- 设置日志聚集服务器地址 -->
<property>  
    <name>yarn.log.server.url</name>  
    <value>http://warehouse102:19888/jobhistory/logs</value>
</property>

<!-- 设置日志保留时间为7天 -->
<property>
    <name>yarn.log-aggregation.retain-seconds</name>
    <value>604800</value>
</property>

</configuration>

配置mapred-site.xml

[scorpion@warehouse102 hadoop]$ vim $HADOOP_HOME/etc/hadoop/mapred-site.xml

<configuration>
	<!-- 指定MapReduce程序运行在Yarn上 -->
    <property>
        <name>mapreduce.framework.name</name>
        <value>yarn</value>
    </property>
    <!-- 历史服务器端地址 -->
<property>
    <name>mapreduce.jobhistory.address</name>
    <value>warehouse102:10020</value>
</property>

<!-- 历史服务器web端地址 -->
<property>
    <name>mapreduce.jobhistory.webapp.address</name>
    <value>warehouse102:19888</value>
</property>

</configuration>

配置workers

[scorpion@warehouse102 hadoop]$ vim $HADOOP_HOME/etc/hadoop/workers

warehouse102
warehouse103
warehouse104

分发后进行第一次启动(需格式化)

[scorpion@warehouse102 hadoop-3.1.3]$ bin/hdfs namenode -format
[scorpion@warehouse102 hadoop-3.1.3]$ sbin/start-dfs.sh

[scorpion@warehouse103 hadoop-3.1.3]$ sbin/start-yarn.sh

Web端查看HDFS的Web页面：http://warehouse102:9870/

Web端查看SecondaryNameNode页面: http://warehouse104:9868/status.html

Hadoop群起脚本

[scorpion@warehouse102 bin]$ cd /home/scorpion/bin
[scorpion@warehouse102 bin]$ vim hadoop-cluster-together.sh

#!/bin/bash
# hadoop群起脚本: > *.sh start
if [ $# -lt 1 ]
then
    echo "No Args Input..."
    exit ;
fi
case $1 in
"start")
        echo " =================== 启动 hadoop集群 ==================="

        echo " --------------- 启动 hdfs ---------------"
        ssh warehouse102 "/opt/module/hadoop-3.1.3/sbin/start-dfs.sh"
        echo " --------------- 启动 yarn ---------------"
        ssh warehouse103 "/opt/module/hadoop-3.1.3/sbin/start-yarn.sh"
        echo " --------------- 启动 historyserver ---------------"
        ssh warehouse102 "/opt/module/hadoop-3.1.3/bin/mapred --daemon start historyserver"
;;
"stop")
        echo " =================== 关闭 hadoop集群 ==================="

        echo " --------------- 关闭 historyserver ---------------"
        ssh warehouse102 "/opt/module/hadoop-3.1.3/bin/mapred --daemon stop historyserver"
        echo " --------------- 关闭 yarn ---------------"
        ssh warehouse103 "/opt/module/hadoop-3.1.3/sbin/stop-yarn.sh"
        echo " --------------- 关闭 hdfs ---------------"
        ssh warehouse102 "/opt/module/hadoop-3.1.3/sbin/stop-dfs.sh"
;;
*)
    echo "Input Args Error..."
;;
esac

[scorpion@warehouse102 bin]$ chmod 777 hadoop-cluster-together.sh