大数据集群软件启动脚本
大家也可以访问我的gitee仓库直接克隆源码大数据软件启动脚本:
包括:hadoop,zookeeper,kafka,flume,hive,maxwell,dolphinscheduler,superset,集群一键启动,集群文件分发,集群执行命令,集群同步时间脚本
hadoop
#!/bin/bash
# hadoop集群启动
if [ $# -eq 0 ]; then
echo "Usage $0 start|stop"
fi
case $1 in
"start"){
echo " -------- 启动 集群 -------"
echo " -------- 启动 hadoop集群 -------"
ssh niit01 "source /etc/profile;start-dfs.sh"
ssh niit03 "source /etc/profile;start-yarn.sh"
ssh niit01 "source /etc/profile;mr-jobhistory-daemon.sh start historyserver"
};;
"stop"){
echo " -------- 停止 集群 -------"
echo " -------- 停止 hadoop集群 -------"
ssh niit01 "source /etc/profile;mr-jobhistory-daemon.sh stop historyserver"
ssh niit03 "source /etc/profile;stop-yarn.sh"
ssh niit01 "source /etc/profile;stop-dfs.sh"
};;
esac
zookeeper
#!/bin/bash
# zookeeper集群启动脚本
echo "Usage: $0 start|stop|status (zookeeper)"
case $1 in
"start"){
for i in niit01 niit02 niit03
do
echo ---------- zookeeper $i 启动 ------------
ssh $i "/opt/onlineedu/zookeeper/bin/zkServer.sh start"
done
};;
"stop"){
for i in niit01 niit02 niit03
do
echo ---------- zookeeper $i 停止 ------------
ssh $i "/opt/onlineedu/zookeeper/bin/zkServer.sh stop"
done
};;
"status"){
for i in niit01 niit02 niit03
do
echo ---------- zookeeper $i 状态 ------------
ssh $i "/opt/onlineedu/zookeeper/bin/zkServer.sh status"
done
};;
esac
kafka
#!/bin/bash
# kafka集群启动
case $1 in
"start"){
for i in niit01 niit02 niit03
do
echo " --------启动 $i Kafka-------"
ssh $i "/opt/onlineedu/kafka/bin/kafka-server-start.sh -daemon /opt/onlineedu/kafka/config/server.properties"
done
};;
"stop"){
for i in niit01 niit02 niit03
do
echo " --------停止 $i Kafka-------"
ssh $i "/opt/onlineedu/kafka/bin/kafka-server-stop.sh stop"
done
};;
esac
flume
#! /bin/bash
# flume采集通道1启动脚本
case $1 in
"start"){
for i in niit01 niit02
do
echo " --------启动 $i 采集flume-------"
ssh $i "nohup /opt/onlineedu/flume/bin/flume-ng agent --conf-file /opt/onlineedu/flume/job/file-flume-kafka.conf --name a1 -Dflume.root.logger=INFO,LOGFILE >/opt/onlineedu/flume/log1.txt 2>&1 &"
done
};;
"stop"){
for i in niit01 niit02
do
echo " --------停止 $i 采集flume-------"
ssh $i "ps -ef | grep file-flume-kafka | grep -v grep |awk '{print \$2}' | xargs -n1 kill -9 "
done
};;
esac
#! /bin/bash
# flume采集通道2启动脚本
case $1 in
"start"){
for i in niit03
do
echo " --------启动 $i 消费flume-------"
ssh $i "nohup /opt/onlineedu/flume/bin/flume-ng agent --conf-file /opt/onlineedu/flume/job/kafka-flume-hdfs.conf --name a1 -Dflume.root.logger=INFO,LOGFILE >/opt/onlineedu/flume/log2.txt 2>&1 &"
done
};;
"stop"){
for i in niit03
do
echo " --------停止 $i 消费flume-------"
ssh $i "ps -ef | grep kafka-flume-hdfs | grep -v grep | awk '{print \$2}' | xargs -n1 kill -9 "
done
};;
esac
#!/bin/bash
# flume采集通道3启动脚本
case $1 in
"start")
echo " --------启动 niit03 业务数据flume-------"
ssh niit03 "nohup /opt/onlineedu/flume/bin/flume-ng agent -n a1 -c /opt/onlineedu/flume/conf -f /opt/onlineedu/flume/job/kafka_to_hdfs_db.conf >/opt/onlineedu/flume/log3.txt >/dev/null 2>&1 &"
;;
"stop")
echo " --------停止 niit03 业务数据flume-------"
ssh niit03 "ps -ef | grep kafka_to_hdfs_db | grep -v grep |awk '{print \$2}' | xargs -n1 kill"
;;
esac
hive
#!/bin/bash
# hive启动
HIVE_LOG_DIR=$HIVE_HOME/logs
mkdir -p $HIVE_LOG_DIR
#检查进程是否运行正常,参数1为进程名,参数2为进程端口
function check_process()
{
pid=$(ps -ef 2>/dev/null | grep -v grep | grep -i $1 | awk '{print $2}')
ppid=$(netstat -nltp 2>/dev/null | grep $2 | awk '{print $7}' | cut -d '/' -f 1)
echo $pid
[[ "$pid" =~ "$ppid" ]] && [ "$ppid" ] && return 0 || return 1
}
function hive_start()
{
metapid=$(check_process HiveMetastore 9083)
cmd="nohup hive --service metastore >$HIVE_LOG_DIR/metastore.log 2>&1 &"
cmd=$cmd" sleep 4; hdfs dfsadmin -safemode wait >/dev/null 2>&1"
[ -z "$metapid" ] && eval $cmd || echo "Metastroe服务已启动"
server2pid=$(check_process HiveServer2 10000)
cmd="nohup hive --service hiveserver2 >$HIVE_LOG_DIR/hiveServer2.log 2>&1 &"
[ -z "$server2pid" ] && eval $cmd || echo "HiveServer2服务已启动"
}
function hive_stop()
{
metapid=$(check_process HiveMetastore 9083)
[ "$metapid" ] && kill $metapid || echo "Metastore服务未启动"
server2pid=$(check_process HiveServer2 10000)
[ "$server2pid" ] && kill $server2pid || echo "HiveServer2服务未启动"
}
case $1 in
"start")
hive_start
;;
"stop")
hive_stop
;;
"restart")
hive_stop
sleep 2
hive_start
;;
"status")
check_process HiveMetastore 9083 >/dev/null && echo "Metastore服务运行正常" || echo "Metastore服务运行异常"
check_process HiveServer2 10000 >/dev/null && echo "HiveServer2服务运行正常" || echo "HiveServer2服务运行异常"
;;
*)
echo Invalid Args!
echo 'Usage: '$(basename $0)' start|stop|restart|status (hiveserver2 & metastore)'
;;
esac
maxwell
#!/bin/bash
# hive启动
HIVE_LOG_DIR=$HIVE_HOME/logs
mkdir -p $HIVE_LOG_DIR
#检查进程是否运行正常,参数1为进程名,参数2为进程端口
function check_process()
{
pid=$(ps -ef 2>/dev/null | grep -v grep | grep -i $1 | awk '{print $2}')
ppid=$(netstat -nltp 2>/dev/null | grep $2 | awk '{print $7}' | cut -d '/' -f 1)
echo $pid
[[ "$pid" =~ "$ppid" ]] && [ "$ppid" ] && return 0 || return 1
}
function hive_start()
{
metapid=$(check_process HiveMetastore 9083)
cmd="nohup hive --service metastore >$HIVE_LOG_DIR/metastore.log 2>&1 &"
cmd=$cmd" sleep 4; hdfs dfsadmin -safemode wait >/dev/null 2>&1"
[ -z "$metapid" ] && eval $cmd || echo "Metastroe服务已启动"
server2pid=$(check_process HiveServer2 10000)
cmd="nohup hive --service hiveserver2 >$HIVE_LOG_DIR/hiveServer2.log 2>&1 &"
[ -z "$server2pid" ] && eval $cmd || echo "HiveServer2服务已启动"
}
function hive_stop()
{
metapid=$(check_process HiveMetastore 9083)
[ "$metapid" ] && kill $metapid || echo "Metastore服务未启动"
server2pid=$(check_process HiveServer2 10000)
[ "$server2pid" ] && kill $server2pid || echo "HiveServer2服务未启动"
}
case $1 in
"start")
hive_start
;;
"stop")
hive_stop
;;
"restart")
hive_stop
sleep 2
hive_start
;;
"status")
check_process HiveMetastore 9083 >/dev/null && echo "Metastore服务运行正常" || echo "Metastore服务运行异常"
check_process HiveServer2 10000 >/dev/null && echo "HiveServer2服务运行正常" || echo "HiveServer2服务运行异常"
;;
*)
echo Invalid Args!
echo 'Usage: '$(basename $0)' start|stop|restart|status (hiveserver2 & metastore)'
;;
esac
dolphinscheduler
#!/bin/bash
# dolphinscheduler集群启动
function node_start()
{
/opt/onlineedu/dolphinscheduler/bin/start-all.sh
/home/niit/bin/xRun.sh jps
}
function node_stop()
{
/opt/onlineedu/dolphinscheduler/bin/stop-all.sh
/home/niit/bin/xRun.sh jps
}
case $1 in
"start")
node_start
;;
"stop")
node_stop
;;
"restart")
node_stop
sleep 10
node_start
;;
*)
esac
#!/bin/bash
# dolphinscheduler单机版启动
function node_start()
{
/opt/onlineedu/dolphinscheduler/bin/dolphinscheduler-daemon.sh start standalone-server
/home/niit/bin/xRun.sh jps
}
function node_stop()
{
/opt/onlineedu/dolphinscheduler/bin/dolphinscheduler-daemon.sh stop standalone-server
/home/niit/bin/xRun.sh jps
}
case $1 in
"start")
node_start
;;
"stop")
node_stop
;;
"restart")
node_stop
sleep 10
node_start
;;
*)
esac
superset
#!/bin/bash
# superset可视化启动
superset_status(){
result=`ps -ef | awk '/gunicorn/ && !/awk/{print $2}' | wc -l`
if [[ $result -eq 0 ]]; then
return 0
else
return 1
fi
}
superset_start(){
source ~/.bashrc
superset_status >/dev/null 2>&1
if [[ $? -eq 0 ]]; then
conda activate superset ; gunicorn --workers 5 --timeout 120 --bind niit01:8787 --daemon 'superset.app:create_app()'
else
echo "superset正在运行"
fi
}
superset_stop(){
superset_status >/dev/null 2>&1
if [[ $? -eq 0 ]]; then
echo "superset未在运行"
else
ps -ef | awk '/gunicorn/ && !/awk/{print $2}' | xargs kill -9
fi
}
case $1 in
start )
echo "启动Superset"
superset_start
;;
stop )
echo "停止Superset"
superset_stop
;;
restart )
echo "重启Superset"
superset_stop
superset_start
;;
status )
superset_status >/dev/null 2>&1
if [[ $? -eq 0 ]]; then
echo "superset未在运行"
else
echo "superset正在运行"
fi
esac
集群一键启动
#!/bin/bash
# 所有集群所有软件启动脚本
function node_start()
{
/home/niit/bin/xCluster.sh start
sleep 15
/home/niit/bin/zk.sh start
sleep 10
/home/niit/bin/kf.sh start
sleep 10
/home/niit/bin/hv.sh start
sleep 5
/home/niit/bin/mxw.sh start
sleep 5
/home/niit/bin/f1.sh start
sleep 5
/home/niit/bin/f2.sh start
sleep 5
/home/niit/bin/f3.sh start
sleep 5
/home/niit/bin/dolphinscheduler-standalone.sh stop
sleep 5
/home/niit/bin/superset.sh start
sleep 5
/home/niit/bin/xRun.sh jps
}
function node_stop()
{
/home/niit/bin/mxw.sh stop
sleep 5
/home/niit/bin/f1.sh stop
sleep 5
/home/niit/bin/f2.sh stop
sleep 5
/home/niit/bin/f3.sh stop
sleep 5
/home/niit/bin/kf.sh stop
sleep 10
/home/niit/bin/kf.sh stop
sleep 10
/home/niit/bin/hv.sh stop
sleep 5
/home/niit/bin/xCluster.sh stop
sleep 15
/home/niit/bin/zk.sh stop
sleep 10
/home/niit/bin/superset.sh stop
sleep 5
/home/niit/bin/dolphinscheduler-standalone.sh stop
sleep 5
/home/niit/bin/xRun.sh jps
}
case $1 in
"start")
node_start
;;
"stop")
node_stop
;;
"restart")
node_stop
sleep 10
node_start
;;
*)
esac
集群文件分发
#!/bin/bash
# 集群分发文件脚本,普通用户权限
#1. 判断参数个数
if [ $# -lt 1 ]
then
echo Not Enough Arguement!
echo Usage: xSync.sh path1 path2..
exit;
fi
#2. 遍历集群所有机器
for host in niit01 niit02 niit03
do
echo ==================== $host ====================
#3. 遍历所有目录,挨个发送
for file in $@
do
#4 判断文件是否存在
if [ -e $file ]
then
#5. 获取父目录
pdir=$(cd -P $(dirname $file); pwd)
#6. 获取当前文件的名称
fname=$(basename $file)
ssh $host "mkdir -p $pdir"
rsync -av $pdir/$fname $host:$pdir
else
echo $file does not exists!
fi
done
done
#!/bin/bash
# 集群分发文件脚本,root权限
#1. 判断参数个数
if [ $# -lt 1 ]
then
echo Not Enough Arguement!
echo Usage: xSync.sh path1 path2 ...
exit;
fi
#2. 遍历集群所有机器
for host in niit01 niit02 niit03
do
echo ==================== $host ====================
#3. 遍历所有目录,挨个发送
for file in $@
do
#4 判断文件是否存在
if [ -e $file ]
then
#5. 获取父目录
pdir=$(cd -P $(dirname $file); pwd)
#6. 获取当前文件的名称
fname=$(basename $file)
sudo ssh $host "mkdir -p $pdir"
sudo rsync -av $pdir/$fname $host:$pdir
else
echo $file does not exists!
fi
done
done
集群执行命令
#/bin/bash
# 集群执行命令脚本
if (( $# == 0 ));then
echo "Usage xRun.sh \"<COMMAND>\""
exit 0
fi
for node in niit01 niit02 niit03
do
echo "======== $node ========"
echo "ssh $node $1"
ssh $node "source /etc/profile;$1"
done
集群同步时间
#!/bin/bash
# 集群时间同步脚本,需要安装chrony到集群中
# 检查参数是否为空
if [ -z "$1" ]; then
echo "Usage: `basename $0` yyyy-MM-dd HH:mm:ss"
exit 1
fi
# 使用date命令将时间字符串转换为日期和时间
# 如果转换失败,则说明时间字符串不合法
if ! date -d "$*" >/dev/null 2>&1; then
echo "Wrong argument for $*"
echo "Usage: `basename $0` yyyy-MM-dd HH:mm:ss"
exit 1
fi
echo ">>>>>>>>>>>> SYNC TIME START >>>>>>>>>>>>"
sum=-1
while [ $sum -ne 0 ]; do
echo set time for niit01 to $1 '>>>'
ssh niit01 "(sudo timedatectl set-ntp false && sudo date -s \"$*\" && sudo timedatectl set-ntp true)"
ok1=$?
echo sync time from niie02 to niit01 '>>>'
ssh niit02 "(sudo timedatectl set-ntp false && sudo timedatectl set-ntp true)"
ok2=$?
echo sync time from niit03 to niit01 '>>>'
ssh niit03 "(sudo timedatectl set-ntp false && sudo timedatectl set-ntp true)"
ok3=$?
sum=`expr $ok1 + $ok2 + $ok3`
if [ $sum -eq 0 ]; then
echo "<<<<<<<<<<<<< SYNC TIME END <<<<<<<<<<<<<"
sleep 5
xRun.sh date
else
echo "sync time failed, will try 10 senconds later"
sleep 10
fi
done