Hadoop生态体系常用脚本

Hadoop生态常用脚本

1.集群内分发脚本(Allsync)

#!/bin/bash
#1.判断参数个数，如果分发参数为空则返回错误
if [ $# -lt 1 ]
then
	echo please give at least one argument
	exit;
fi
#2.遍历集群所有机器
for $host in hdpA hdpB hdpC
do
	echo ============= $host ============
#3.遍历所有目录，逐个发送
for file in $@
do
	#4.判断文件是否存在
	if [ -e $file ]
	then
		#5.获取文件父目录
		pdir=$(cd -p $(dirname $file);pwd)
		#6.获取当前文件名
		fname=$(basename $file)
		ssh $host "mkdir -p $pdir"
		rsync -av $pdir/$fname $host:$pdir
	else
		echo $file does not exist !
	fi
done

2. 查看三台服务器java进程脚本 （jpsall）

#!/bin/bash
#1.遍历三台服务器
for host in hdpA hdpB hdpC
do
	echo ============= $host ================
	ssh $host jps $@ | grep -v jps
done

3. hadoop集群启动脚本 （hdp.sh）

#!/bin/bash
#1.判断输入参数是否为空
if [ $# -lt 1 ]
then
    echo "No Args Input..."
    exit ;
fi
#2.根据脚本输入的参数为start/stop来执行相应命令
case $1 in
"start")
        echo " =================== 启动 hadoop集群 ==================="

        echo " --------------- 启动 hdfs ---------------"
        ssh hdpA "$HADOOP_HOME/sbin/start-dfs.sh"
        echo " --------------- 启动 yarn ---------------"
        ssh hdpB "$HADOOP_HOME/sbin/start-yarn.sh"
        echo " --------------- 启动 historyserver ---------------"
        ssh hdpC "$HADOOP_HOME/bin/mapred --daemon start historyserver"
;;
"stop")
        echo " =================== 关闭 hadoop集群 ==================="

        echo " --------------- 关闭 historyserver ---------------"
        ssh hdpC "$HADOOP_HOME/bin/mapred --daemon stop historyserver"
        echo " --------------- 关闭 yarn ---------------"
        ssh hdpB "$HADOOP_HOME/sbin/stop-yarn.sh"
        echo " --------------- 关闭 hdfs ---------------"
        ssh hdpA "$HADOOP_HOME/sbin/stop-dfs.sh"
;;
*)
    echo "输入参数错误"
;;
esac

4. hive服务启动脚本（hiveservices.sh）

此脚本用于管理metastore和hiveserver2服务的启动和关闭

#!/bin/bash
HIVE_LOG_DIR=$HIVE_HOME/logs
if [ ! -d $HIVE_LOG_DIR ]
then
	mkdir -p $HIVE_LOG_DIR
fi
#检查进程是否运行正常，参数1为进程名，参数2为进程端口
function check_process()
{
    pid=$(ps -ef 2>/dev/null | grep -v grep | grep -i $1 | awk '{print $2}')
    ppid=$(netstat -nltp 2>/dev/null | grep $2 | awk '{print $7}' | cut -d '/' -f 1)
    echo $pid
    [[ "$pid" =~ "$ppid" ]] && [ "$ppid" ] && return 0 || return 1
}

function hive_start()
{
    metapid=$(check_process HiveMetastore 9083)
    cmd="nohup hive --service metastore >$HIVE_LOG_DIR/metastore.log 2>&1 &"
    cmd=$cmd" sleep 4; hdfs dfsadmin -safemode wait >/dev/null 2>&1"
    [ -z "$metapid" ] && eval $cmd || echo "Metastroe服务已启动"
    server2pid=$(check_process HiveServer2 10000)
    cmd="nohup hive --service hiveserver2 >$HIVE_LOG_DIR/hiveServer2.log 2>&1 &"
    [ -z "$server2pid" ] && eval $cmd || echo "HiveServer2服务已启动"
}

function hive_stop()
{
    metapid=$(check_process HiveMetastore 9083)
    [ "$metapid" ] && kill $metapid || echo "Metastore服务未启动"
    server2pid=$(check_process HiveServer2 10000)
    [ "$server2pid" ] && kill $server2pid || echo "HiveServer2服务未启动"
}

case $1 in
"start")
    hive_start
    ;;
"stop")
    hive_stop
    ;;
"restart")
    hive_stop
    sleep 2
    hive_start
    ;;
"status")
    check_process HiveMetastore 9083 >/dev/null && echo "Metastore服务运行正常" || echo "Metastore服务运行异常"
    check_process HiveServer2 10000 >/dev/null && echo "HiveServer2服务运行正常" || echo "HiveServer2服务运行异常"
    ;;
*)
    echo Invalid Args!
    echo 'Usage: '$(basename $0)' start|stop|restart|status'
    ;;
esac

5. zookeeper集群启动停止脚本（zk.sh）

#! /bin/bash
#根据脚本参数来判断执行哪个命令
case $1 in
"start"){
    for i in hdpA hdpB hdpC
    do
        echo " --------启动 $i Kafka-------"
        ssh $i "$ZOOKEEPER_HOME/bin/kafka-server-start.sh -daemon $ZOOKEEPER_HOME/config/server.properties"
    done
};;
"stop"){
    for i in hdpA hdpB hdpC
    do
        echo " --------停止 $i Kafka-------"
        ssh $i "$ZOOKEEPER_HOME/bin/kafka-server-stop.sh stop"
    done
};;
esac

6. kafka集群启动停止脚本（kf.sh）

#!/bin/bash
case $1 in
"start")
   for i in hdpA hdpB hdpC
   do
   echo "============== $i kafka============== "
   ssh $i "$KAFKA_HOME/bin/kafka-server-start.sh -daemon $KAFKA_HOME/config/server.properties"
   done
;;
"stop")
   for i in hadoop102 hadoop103 hadoop104
   do
   echo "============== $i kafka============== "
   ssh $i "$KAFKA_HOME/bin/kafka-server-stop.sh"
   done
esac