Hadoop相关脚本 - 仅供参考

eagle脚本

#!/bin/bash
if [ $# -lt 1 ]
  then
  echo "No Args Input Error !"
  exit
fi

case $1 in
"start")
  /opt/module/eagle/bin/ke.sh start
;;
"stop")
  /opt/module/eagle/bin/ke.sh stop
;;
esac

flume.servers脚本

#!/bin/bash
if [ $# -lt 1 ]
  then
    echo "No Args Input Error !"
    exit
fi

case $1 in
"start")
  echo "================== start flume.servers ==================="
  echo "starting..."

  for i in `cat /opt/module/hadoop-3.1.3/etc/hadoop/workers`
  do
    ssh $i 'sudo systemctl start gmond'
    if [ $i == LIFE ]
      then
      ssh LIFE 'sudo systemctl start httpd'
      ssh LIFE 'sudo systemctl start gmetad'
    fi
  done
;;
"stop")
  echo "================== stop flume.servers ==================="
  echo "stop..."

  for i in `cat /opt/module/hadoop-3.1.3/etc/hadoop/workers`
  do
    ssh $i 'sudo systemctl stop gmond'
    if [ $i == LIFE ]
      then
      ssh LIFE 'sudo systemctl stop httpd'
      ssh LIFE 'sudo systemctl stop gmetad'
    fi
  done
;;
"status")
  for i in `cat /opt/module/hadoop-3.1.3/etc/hadoop/workers`
  do
    echo "==================== status $i flumes.servers =================="
    ssh $i 'sudo systemctl status gmond'
    if [ $i == LIFE ]
      then
      ssh LIFE 'sudo systemctl status httpd'
      ssh LIFE 'sudo systemctl status gmetad'
    fi
  done
;;
"restart")
  flume.servers stop
  flume.servers start
;;
*)
  echo "Input Args Error !"
;;
esac

jobHistory命令

启动 : mapred --daemon start historyserver
关闭 : mapred --daemon stop historyserver

Kafka脚本

#!/bin/bash
if [ $# -lt 1 ]
 then
   echo "No Args Input Error"
   exit
fi

case $1 in
  "start")
    for i in `cat /opt/module/hadoop-3.1.3/etc/hadoop/workers`
      do
        echo "==========start $i kafka=========="
        ssh $i '$KAFKA_HOME/bin/kafka-server-start.sh -daemon /opt/module/kafka/config/server.properties'
      done
  ;;
  "stop")
    for i in `cat /opt/module/hadoop-3.1.3/etc/hadoop/workers`
      do
        echo "==========stop $i kafka=========="
        ssh $i '$KAFKA_HOME/bin/kafka-server-stop.sh'
    done
  ;;
  *)
    echo "Input Args Error"
  ;;
esac

log脚本

#!/bin/bash
#设置 lofe101 和lofe102 为日志服务器
for i in lofe101 lofe102;
do
    echo "========== $i =========="
    ssh $i "cd /opt/module/applog/; java -jar gmall2020-mock-log-2020-04-01.jar >/dev/null 2>&1 &"
done

集群启停脚本 mycluster

#!/bin/bash
if [ $# -lt 1 ]
 then
  echo "No Args Input Error!"
  exit
fi

case $1 in
"start")
 echo "=============== start hdfs ==============="
 ssh LIFE /opt/module/hadoop-3.1.3/sbin/start-dfs.sh
 echo "=============== start yarn ==============="
 ssh LIVE /opt/module/hadoop-3.1.3/sbin/start-yarn.sh
;;
"stop")
 echo "=============== stop hdfs ==============="
 ssh LIFE /opt/module/hadoop-3.1.3/sbin/stop-dfs.sh
 echo "=============== stop yarn ==============="
 ssh LIVE /opt/module/hadoop-3.1.3/sbin/stop-yarn.sh
;;
#################### restart #######################
"restart")
 mycluster stop
 mycluster start
;;
*)
 echo "Input Args Error!"
;;
esac

查看集群jps myjps

#!/bin/bash
for i in LIFE LIVE LOVE
do
 echo "=============== $i JPS ==============="
 ssh $i /opt/module/jdk1.8.0_212/bin/jps
done

Web从HDFS删除文件许可配置

配置core-site.xml

<!-- 授权web界面增删改查(认定登录web端的用户为atguigu) -->
    <property>
        <name>hadoop.http.staticuser.user</name>
        <value>atguigu</value>
    </property>

群发脚本 xsync

#!/bin/bash
# 1. 判断参数个数
if [ $# -lt 1 ]
then
 echo Not Enough Arguement!
 exit;
fi
# 2. 遍历集群所有机器
for host in LIVE LOVE
do
 echo =============== $host ===============
 # 3. 遍历所有目录, 挨个发送
 for file in $@
 do
  # 4. 判断文件是否存在
  if [ -e $file ]
  then
   # 5. 获取父目录
   pdir=$(cd -P $(dirname $file); pwd)
   # 6. 获取当前文件的名称
   fname=$(basename $file)
   ssh $host "mkdir -p $pdir"
   rsync -av $pdir/$fname $host:$pdir
  else
   echo $file does not exits!
  fi
 done
done

Zookeeper启停脚本

#!/bin/bash
if [ $# -lt 1 ]
  then
    echo "No Args Input Error !"
    exit
fi

case $1 in
"start")
  for i in `cat /opt/module/hadoop-3.1.3/etc/hadoop/workers`
  do
    echo "================ start $i zookeeper ==============="
    ssh $i '/opt/module/zookeeper-3.5.7/bin/zkServer.sh start'
  done
;;
"stop")
  for i in `cat /opt/module/hadoop-3.1.3/etc/hadoop/workers`
  do
    echo "================ stop $i zookeeper ==============="
    ssh $i '/opt/module/zookeeper-3.5.7/bin/zkServer.sh stop'
  done
;;
"status")
  for i in `cat /opt/module/hadoop-3.1.3/etc/hadoop/workers`
  do
    echo "================ status $i zookeeper =============="
    ssh $i '/opt/module/zookeeper-3.5.7/bin/zkServer.sh status'
  done
;;
*)
  echo "Input Args Error !"
;;
esac