采集项目各组件脚本代码

hadoop启停脚本

vim hdp.sh

输入如下内容：

#!/bin/bash

if [ $# -lt 1 ]

then

    echo "No Args Input..."

    exit ;

fi

case $1 in

"start")

        echo " =================== 启动 hadoop集群 ==================="

        echo " --------------- 启动 hdfs ---------------"

        ssh hadoop102 "/opt/module/hadoop/sbin/start-dfs.sh"

        echo " --------------- 启动 yarn ---------------"

        ssh hadoop103 "/opt/module/hadoop/sbin/start-yarn.sh"

        echo " --------------- 启动 historyserver ---------------"

        ssh hadoop102 "/opt/module/hadoop/bin/mapred --daemon start historyserver"

;;

"stop")

        echo " =================== 关闭 hadoop集群 ==================="

        echo " --------------- 关闭 historyserver ---------------"

        ssh hadoop102 "/opt/module/hadoop/bin/mapred --daemon stop historyserver"

        echo " --------------- 关闭 yarn ---------------"

        ssh hadoop103 "/opt/module/hadoop/sbin/stop-yarn.sh"

        echo " --------------- 关闭 hdfs ---------------"

        ssh hadoop102 "/opt/module/hadoop/sbin/stop-dfs.sh"

;;

*)

    echo "Input Args Error..."

;;

esac

ZK集群启动停止脚本

im zk.sh

       在脚本中编写如下内容

#!/bin/bash

case $1 in

"start"){

    for i in hadoop102 hadoop103 hadoop104

    do

        echo ---------- zookeeper $i 启动 ------------

       ssh $i "/opt/module/zookeeper/bin/zkServer.sh start"

    done

};;

"stop"){

    for i in hadoop102 hadoop103 hadoop104

    do

        echo ---------- zookeeper $i 停止 ------------   

       ssh $i "/opt/module/zookeeper/bin/zkServer.sh stop"

    done

};;

"status"){

    for i in hadoop102 hadoop103 hadoop104

    do

        echo ---------- zookeeper $i 状态 ------------   

       ssh $i "/opt/module/zookeeper/bin/zkServer.sh status"

    done

};;

esac

kafka集群启停脚本

vim kf.sh

脚本如下：

#! /bin/bash

case $1 in

"start"){

    for i in hadoop102 hadoop103 hadoop104

    do

        echo " --------启动 $i Kafka-------"

        ssh $i "/opt/module/kafka/bin/kafka-server-start.sh -daemon /opt/module/kafka/config/server.properties"

    done

};;

"stop"){

    for i in hadoop102 hadoop103 hadoop104

    do

        echo " --------停止 $i Kafka-------"

        ssh $i "/opt/module/kafka/bin/kafka-server-stop.sh "

    done

};;

esac

flume启停脚本

vim f1.sh

       在脚本中填写如下内容

#!/bin/bash

case $1 in

"start"){

        for i in hadoop102 hadoop103

        do

                echo " --------启动 $i 采集flume-------"

                ssh $i "nohup /opt/module/flume/bin/flume-ng agent -n a1 -c /opt/module/flume/conf/ -f /opt/module/flume/job/file_to_kafka.conf >/dev/null 2>&1 &"

        done

};;

"stop"){

        for i in hadoop102 hadoop103

        do

                echo " --------停止 $i 采集flume-------"

                ssh $i "ps -ef | grep file_to_kafka | grep -v grep |awk  '{print \$2}' | xargs -n1 kill -9 "

        done

};;

esac

vim f2.sh

       在脚本中填写如下内容

#!/bin/bash

case $1 in

"start")

        echo " --------启动 hadoop104 日志数据flume-------"

        ssh hadoop104 "nohup /opt/module/flume/bin/flume-ng agent -n a1 -c /opt/module/flume/conf -f /opt/module/flume/job/kafka_to_hdfs_log.conf >/dev/null 2>&1 &"

;;

"stop")

        echo " --------停止 hadoop104 日志数据flume-------"

        ssh hadoop104 "ps -ef | grep kafka_to_hdfs_log | grep -v grep |awk '{print \$2}' | xargs -n1 kill -9"

;;

esac

vim f3.sh

       在脚本中填写如下内容

#!/bin/bash

case $1 in

"start")

        echo " --------启动 hadoop104 业务数据flume-------"

        ssh hadoop104 "nohup /opt/module/flume/bin/flume-ng agent -n a1 -c /opt/module/flume/conf -f /opt/module/flume/job/kafka_to_hdfs_db.conf >/dev/null 2>&1 &"

;;

"stop")

        echo " --------停止 hadoop104 业务数据flume-------"

        ssh hadoop104 "ps -ef | grep kafka_to_hdfs_db | grep -v grep |awk '{print \$2}' | xargs -n1 kill -9"

;;

esac

Maxwell启停脚本

vim mxw.sh

（2）脚本内容如下

#!/bin/bash

MAXWELL_HOME=/opt/module/maxwell

status_maxwell(){

    result=`ps -ef | grep com.zendesk.maxwell.Maxwell | grep -v grep | wc -l`

    return $result

}

start_maxwell(){

    status_maxwell

    if [[ $? -lt 1 ]]; then

        echo "启动Maxwell"

        $MAXWELL_HOME/bin/maxwell --config $MAXWELL_HOME/config.properties --daemon

    else

        echo "Maxwell正在运行"

    fi

}

stop_maxwell(){

    status_maxwell

    if [[ $? -gt 0 ]]; then

        echo "停止Maxwell"

        ps -ef | grep com.zendesk.maxwell.Maxwell | grep -v grep | awk '{print $2}' | xargs kill -9

    else

        echo "Maxwell未在运行"

    fi

}

case $1 in

    start )

        start_maxwell

    ;;

    stop )

        stop_maxwell

    ;;

    restart )

       stop_maxwell

       start_maxwell

    ;;

esac

采集通道启动/停止脚本

vim cluster.sh

       在脚本中填写如下内容

#!/bin/bash

case $1 in

"start"){

        echo ================== 启动 集群 ==================

        #启动 Zookeeper集群

        zk.sh start

        #启动 Hadoop集群

        hdp.sh start

        #启动 Kafka采集集群

        kf.sh start

        #启动采集 Flume

        f1.sh start

#启动日志消费 Flume

        f2.sh start

#启动业务消费 Flume

        f3.sh start

#启动 maxwell

        mxw.sh start

        };;

"stop"){

        echo ================== 停止 集群 ==================

#停止 Maxwell

        mxw.sh stop

#停止 业务消费Flume

        f3.sh stop

#停止 日志消费Flume

        f2.sh stop

#停止 日志采集Flume

        f1.sh stop

        #停止 Kafka采集集群

        kf.sh stop

        #停止 Hadoop集群

        hdp.sh stop

        #停止 Zookeeper集群

        zk.sh stop

};;

esac

各脚本增加执行权限