Hadoop常用脚本(更新中)

Hadoop常用脚本

1.xsync同步脚本

#!/bin/bash
#1. 判断参数个数
if [ $# -lt 1 ]
then
        echo Not Enough Arguement!
        exit;
fi

#2. 遍历集群所有机器
for host in hadoop102 hadoop103 hadoop104
do
        echo ====================  $host  ====================
        #3. 遍历所有目录，挨个发送
        for file in $@
        do
                #4. 判断文件是否存在
                if [ -e $file ]
                        then
                                #5. 获取父目录
                                pdir=$(cd -P $(dirname $file);pwd)
                                #6. 获取当前文件的名称
                                fname=$(basename $file)
                                ssh $host "mkdir -p $pdir"
                                rsync -av $pdir/$fname $host:$pdir
                        else
                                echo $file does not exists!
                fi
        done
done

使用

[student@hadoop102 hadoop-3.1.3]$ xsync hadoop/  --同步hadoop目录下的所有文件及内容
xsync文件内容(此文件存放于操作用户的~/bin中，并附加x执行权限)

2.myhadoop.sh集群启动/停止脚本

(hdfs、yarn、historyserver)

#!/bin/bash

if [ $# -lt 1 ]
then
    echo "No Args Input..."
    exit ;
fi

case $1 in
"start")
        echo " =================== 启动 hadoop集群 ==================="

        echo " --------------- 启动 hdfs ---------------"
        ssh hadoop102 "/opt/module/hadoop-3.1.3/sbin/start-dfs.sh"
        echo " --------------- 启动 yarn ---------------"
        ssh hadoop103 "/opt/module/hadoop-3.1.3/sbin/start-yarn.sh"
        echo " --------------- 启动 historyserver ---------------"
        ssh hadoop102 "/opt/module/hadoop-3.1.3/bin/mapred --daemon start historyserver"
;;
"stop")
        echo " =================== 关闭 hadoop集群 ==================="

        echo " --------------- 关闭 historyserver ---------------"
        ssh hadoop102 "/opt/module/hadoop-3.1.3/bin/mapred --daemon stop historyserver"
        echo " --------------- 关闭 yarn ---------------"
        ssh hadoop103 "/opt/module/hadoop-3.1.3/sbin/stop-yarn.sh"
        echo " --------------- 关闭 hdfs ---------------"
        ssh hadoop102 "/opt/module/hadoop-3.1.3/sbin/stop-dfs.sh"
;;
*)
    echo "Input Args Error..."
;;
esac

使用

[student@hadoop102 hadoop-3.1.3]$ myhadoop.sh stop
xsync文件内容(此文件存放于操作用户的~/bin中，并附加x执行权限)

3.jpsall脚本查看各节点组件

#!/bin/bash

for host in hadoop102 hadoop103 hadoop104
do
        echo =============== $host ===============
        ssh $host jps 
done

4.kafka群启动停止脚本

#!/bin/bash

case $1 in
"start")
        for i in hadoop102 hadoop103 hadoop104
        do
                echo "--- 启动 $i kafka ---"
                ssh $i "/opt/module/kafka_2.12-3.0.0/bin/kafka-server-start.sh -daemon /opt/module/kafka_2.12-3.0.0/config/server.properties"
        done
;;
"stop")
        for i in hadoop102 hadoop103 hadoop104
        do
                echo "--- 停止 $i kafka ---"
                ssh $i "/opt/module/kafka_2.12-3.0.0/bin/kafka-server-stop.sh"
        done
;;
esac

5.zookeeper群启动停止脚本

#!/bin/bash
case $1 in
"start"){
        for i in hadoop102 hadoop103 hadoop104
        do
        echo ---------- zookeeper $i 启动 ------------
                ssh $i "source /etc/profile; /opt/module/zookeeper-3.7.0/bin/zkServer.sh start"
        done
};;
"stop"){
        for i in hadoop102 hadoop103 hadoop104
        do
        echo ---------- zookeeper $i 停止 ------------
                ssh $i "source /etc/profile; /opt/module/zookeeper-3.7.0/bin/zkServer.sh stop"
        done
};;
"status"){
        for i in hadoop102 hadoop103 hadoop104
        do
        echo ---------- zookeeper $i 状态 ------------
                ssh $i "source /etc/profile; /opt/module/zookeeper-3.7.0/bin/zkServer.sh status"
        done
};;
esac

使用

[student@hadoop102 hadoop-3.1.3]$ jpsall
xsync文件内容(此文件存放于操作用户的~/bin中，并附加x执行权限)