大数据运维 fstab磁盘自动挂载脚本

脚本自写，可根据业务自行调整：脚本是将除去系统盘把所有的磁盘均挂载到/mnt/storage**下

 1.获取系统识别磁盘和已经挂载磁盘、mnt下挂载文件目录

2.遍历比较已经挂载的磁盘与fdisk获取的系统识别的磁盘、比较df -h已经挂载的mnt下的目录和ls -l 获取的mnt下目录，进而获取没有挂载的disk和未挂载的mnt下的目录

3.获取没有挂载的disk和未挂载的mnt下的目录，循环格盘挂盘

4.BigDataDev 方法为定制，不需要可以删除

#!/bin/bash
function checkDev(){
  echo "运行环境监测"
  needmkdirfile=(`lsblk -b | grep disk | sort -nk4 | awk '{print $1}'`)
  needmkdirfilesum=${#needmkdirfile[@]}
  for i in `seq -w 0 40`
    do
        if [[ 10#$i -lt $needmkdirfilesum-1 ]];
          then
          mkdir -p /mnt/storage$i > /dev/null 2>&1
              if [ $? -eq 1 ];
              then
                echo "磁盘挂载目录/mnt/storage$i异常"
                exit 0
              fi
          else
              break
        fi
  done
}

function BigDataDev(){
  HostName=`hostname`
  while [[ $HostName == *datanode* && ${needmountfile[i]} == storage00 ]]; do
      echo "创建yarn文件环境"
    mkdir -p /mnt/storage00/logs
    mkdir -p /mnt/storage00/datum
    mkdir -p /mnt/storage00/logs/yarn
    mkdir -p /mnt/storage00/yarn-nm-recovery/
    mkdir -p /sys/fs/cgroup/cpu/hadoop-yarn
    chown -R yarn:hadoop /mnt/storage00/logs/ /mnt/storage00/datum/ /mnt/storage00/yarn-nm-recovery/ /sys/fs/cgroup/cpu/hadoop-yarn/
      break
  done
  while [[ $HostName == *datanode* && ${needmountfile[i]} != storage00 ]]; do
          echo "${needmountdisk[i]}磁盘挂载检查"
          file=${needmountfile[i]}
          filenum=`echo ${file#*e}`
          BeforeMountfilenum=`printf "%02d" $((10#$filenum-1))`
          BeforeMountfile=storage$BeforeMountfilenum
          echo "备份hdfs-site.xml到/tmp/"
          rsync -azrL --delete --force /etc/hadoop-datanode/hdfs-site.xml /tmp/
          echo "修改${needmountfile[i]}目录权限"
          chmod -R 775 /mnt/${needmountfile[i]} && chown -R hdfs:hadoop /mnt/${needmountfile[i]}
        mkdir -p /mnt/${needmountfile[i]}/logs
        chown -R yarn:hadoop /mnt/${needmountfile[i]}/logs
          echo "修改hdfs-site.xml配置文件"
          HDFSDir=`cat /etc/hadoop-datanode/hdfs-site.xml | grep -A2 -w "dfs.datanode.data.dir" | grep mnt | sed -e "s/storage[0-9][0-9]/${needmountfile[i]}/g"`
          ip=`cat /etc/network/interfaces | grep address | awk '{print $2}'`
          su - hdfs -c "kinit -kt /etc/security/keytabs/dn.service.keytab `klist -k /etc/security/keytabs/dn.service.keytab | uniq | grep @BILIBILI.CO |awk '{print $2}'`"
          if [[ ${needmountfile[i]} == storage01 ]];
          then
                  sed -i "/\/mnt\/storage02\/datum\/datanode,/i\\$HDFSDir" /etc/hadoop-datanode/hdfs-site.xml
                 # su - hdfs -c "hdfs dfsadmin -reconfig datanode $ip:50020 start"
                 # su - hdfs -c "hdfs dfsadmin -reconfig datanode $ip:50020 status"
          else
                  sed -i "/\/mnt\/$BeforeMountfile\/datum\/datanode,/a\\$HDFSDir" /etc/hadoop-datanode/hdfs-site.xml
                 # su - hdfs -c "hdfs dfsadmin -reconfig datanode $ip:50020 start"
                 # su - hdfs -c "hdfs dfsadmin -reconfig datanode $ip:50020 status"
          fi
          echo "备份yarn-site.xml到/tmp/"
          rsync -azrL --delete --force /etc/hadoop-datanode/yarn-site.xml /tmp/
          needyarnfile=`cat /etc/hadoop-nodemanager/yarn-site.xml | grep -A 2 yarn.nodemanager.storage.type.local-dirs | grep -v value | grep mnt |sed -e "s/storage[0-9][0-9]/${needmountfile[i]}/g"`
          yarnfile=`cat /etc/hadoop-nodemanager/yarn-site.xml | grep -A 15 yarn.nodemanager.storage.type.local-dirs | grep ${needmountfile[i]}`
          if [[ ! -n $yarnfile ]];
          then
          echo "修改yarn-site.xml配置文件"
                  if [[ ${needmountfile[i]} == storage01 ]];
                  then
              sed -i "/\/mnt\/storage02\/logs\/yarn\/nm\/localdir,/i\\$needyarnfile" /etc/hadoop-nodemanager/yarn-site.xml
                  else
                      sed -i "/\/mnt\/$BeforeMountfile\/logs\/yarn\/nm\/localdir,/a\\$needyarnfile" /etc/hadoop-nodemanager/yarn-site.xml
                  fi
                  tiers=`cat /etc/hadoop-nodemanager/yarn-site.xml | grep -A 1 yarn.nodemanager.local-dirs-tiers | grep value | tr -cd 0-9`
                  sed -i "s/$tiers/011111111111111/g"  /etc/hadoop-nodemanager/yarn-site.xml
          #bash /data/script/hadoop-nodemanager/nodemanager-restart.sh
      else
             break
          fi
          break
  done
  while [[ $HostName == *kafka* ]]; do
          echo "创建kafka文件环境"
          mkdir -p /mnt/${needmountfile[i]}/datum
          mkdir -p /mnt/${needmountfile[i]}/datum/kafka_data/
          chmod -R 770 /mnt/${needmountfile[i]} && chown -R kafka:hadoop /mnt/${needmountfile[i]}
          break
  done
}

function getdiskfs(){
    fs=`mount | grep -w /dev | grep mnt | sort -k 3| awk '{print $5}' | awk 'END {print}'`
    if [[ $fs == xfs ]]; then
        echo "自动获取磁盘文件系统格式为:$fs"
        mkfs.xfs -f ${needmountdisk[i]} >> /dev/null 2<&1
        fs=xfs
    elif [[ $fs == ext4 ]]; then
        echo "自动获取磁盘文件系统格式为:$fs"
        mkfs.ext4  ${needmountdisk[i]} >> /dev/null 2<&1
        fs=ext4
    else
        echo “自动获取文件系统格式$fs不是xfs和ext4，强制格盘为xfs”
        mkfs.xfs -f ${needmountdisk[i]} >> /dev/null 2<&1
        fs=xfs
    fi
    echo "盘符${needmountdisk[i]}格式化完成"
}
function Startmount(){
    echo "备份/etc/fstab到/tmp"
    rsync -azrL --delete --force /etc/fstab /tmp/
    NEWUUID=`blkid | grep -w ${needmountdisk[i]} |awk '{print $2}' | awk -F '=' '{print $2}'`
    OLDUUID=`cat /etc/fstab | grep ${needmountfile[i]} | awk '{print $1}' | awk -F '=' '{print $2}'`
    if [ ! -n "$OLDUUID" ];
    then
      echo "添加${needmountdisk[i]}磁盘UUID到/etc/fatab中需挂载的文件/mnt/${needmountfile[i]}"
      echo "UUID="$NEWUUID"   /mnt/${needmountfile[i]}        $fs      defaults,nofail,noatime,nodiratime     0 0  #${needmountdisk[i]}" >> /etc/fstab
      echo "UUID="$NEWUUID"   /mnt/${needmountfile[i]}        $fs      defaults,nofail,noatime,nodiratime     0 0  #${needmountdisk[i]} "
      echo "磁盘挂载到${needmountfile[i]}文件夹"
      mount -a
    else
      echo "添加${needmountdisk[i]}磁盘UUID到/etc/fatab中需挂载的文件/mnt/${needmountfile[i]}"
      sed -i "s/$OLDUUID/$NEWUUID/" /etc/fstab
      echo "磁盘挂载到${needmountfile[i]}文件夹"
      mount -a
    fi
}
checkDev
systemdisk=`fdisk -l | grep -i Linux | awk '{print $1}'| head -n 1 | sed 's/.$//'`
alldisk=(`fdisk -l | grep -i disk | grep dev | sort -nk5 |  awk '{print $2}' | awk -F ":" '{print $1}'| grep -Ev "(docker|loop)" | grep -v -w $systemdisk`)
fsdisk=(`df -h  | grep mnt | awk '{print $1}'`)
allfilesum=`ls /mnt/ | grep storage| wc -l`
mountfilesum=`df -h | grep mnt | wc -l`
allfile=(`ls /mnt/ | grep storage`)
mountfile=(`df -h | grep mnt | awk '{print $6}' | sort -k 1`)
declare -a needmountfile
t=0
flag=0
for nomountfile in "${allfile[@]}"
do
    for file in "${mountfile[@]}"
    do
        if [[ "/mnt/${nomountfile}" == "${file}" ]]; then
            flag=1
            break
        fi
    done
    if [[ $flag -eq 0 ]]; then
        needmountfile[t]=$nomountfile
        t=$((t+1))
    else
        flag=0
    fi
done
declare -a needmountdisk
t=0
flag=0
for mountdisk in "${alldisk[@]}"
do
    for disk in "${fsdisk[@]}"
    do
        if [[ "${mountdisk}" == "${disk}" ]]; then
            flag=1
            break
        fi
    done
    if [[ $flag -eq 0 ]]; then
        needmountdisk[t]=$mountdisk
        t=$((t+1))
    else
        flag=0
    fi
done
mountdisknum=${#needmountdisk[@]}
if [[ $mountdisknum -ge 1 ]]; then
        for (( i = 0; i <= $mountdisknum-1 ; i++ )); do
                echo "----------Begin----------"
                echo "磁盘${needmountdisk[i]}开始格式化"
                getdiskfs
                if [ $? -eq 0 ];
                then
                   Startmount
                   echo "/mnt/${needmountfile[i]}磁盘挂载检查"
                   if mountpoint -q /mnt/${needmountfile[i]}
                   then
                       BigDataDev
                       echo "----------End----------"
                   else
                       echo "${needmountfile[i]}挂载失败"
                       echo "/etc/fatab备份还原"
                       rsync -azrL --delete --force /tmp/fstab /etc/
                       echo "----------End----------"
                       exit 0
                   fi
                else
                        echo "${needmountdisk[i]}磁盘格式化失败"
                        break
                fi
        done
else
        echo “磁盘均已挂载”
        echo “请检查硬盘是否被系统识别”
        exit 0
fi