【Shell脚本】运维常用shell脚本集锦和分析

1. 日志压缩

1.1 简化版

#!/bin/bash
while true
do    
        startTime=`date +%H`    
        if [[ 10#$startTime -eq '01' ]] ; then       
            find /APL/logs  -name "mac-web-marketing-????-??-??-?*.log" | xargs gzip;
            find /APL/logs  -name "mac-web-marketing_error-????-??-??-?*.log" | xargs gzip;
            find /APL/logs  -name "mac-web-marketing-????-??-??-?*.log.gz"  -mtime +30 -exec rm -rf {} \;
            find /APL/logs  -name "mac-web-marketing_error-????-??-??-?*.log.gz"  -mtime +30 -exec rm -rf {} \;
        fi
        sleep 3600
done

#!/bin/bash

# 日志名
logName=$1
# 保留多少天日志
rmTime=$2
# 执行策略：1，每天执行  2，每小时执行
gzipTime=$3

if [[ $gzipTime -eq 1 ]] ; then
    startTime="date +%H"
    sleepTime=3600
elif [[ $gzipTime -eq 2 ]] ; then
    startTime="date +%M"
    sleepTime=60
fi
echo `date +%F" "%T`" 执行脚本 删除策略保留" $2"天"
while true
do
    date=`${startTime}`
    if [[ 10#$date -eq 10 ]] ; then
        file=`find ./ -type f -name "${logName}" -mmin +60 | grep -v gz`
        echo $file

        for log in $file
        do
            gzip $log
        done
        echo $file        
        find ./ -type f -name "${logName}.gz" -mtime +${rmTime} -exec rm -rf {} \;
        
    fi
   sleep $sleepTime
done

• 关于{}和\;的含义，请见文章；
• 关于 value too great for base (error token is "08") ，请见文章；
• 关于 mtime 、 ctime 等含义，请见文章。

2. 全面版

# cd /APL/logs; nohup sh backLog.sh >> backLog.out 2>&1 &

#Please make sure $logs and $cspLogs path exists.
logs=/APL/logs
# magpie流控日志
cspLogs=~/logs/csp
# 打包几天前的日志, 最小取1
# 仅适用magpie日志, 应用日志每小时清理一次
zipDaysAgo=1
# 删除几天前的压缩文件, 最小取1
rmDaysAgo=3
# 执行时间点, 单位24小时制
execTime="01"
# 执行分钟数, 取值00-60
execMin=50
# 固定值，每天秒数
daySecond=`expr 24 \* 60 \* 60`


###clean app log
cleanAppLog() {
    echo `date "+%Y-%m-%d %H:%M:%S"` clean app log begin
    #zip zipDaysAgo log
    cd $logs
    curDirs=`find . -maxdepth 1 -type d  -regex "^\.\/[0-9]+\-.*$"`
    for curDir in ${curDirs[*]}; do
        curDirDate=`date -d "${curDir:2}" +%s`
        diffDate=`expr $1 - ${curDirDate}`
        diffDateNum=`expr ${diffDate} / ${daySecond}`
        if [ ${diffDateNum} -gt ${zipDaysAgo} ]; then
            echo "tar dir ${curDir}"
            tar -zcf ${curDir}.tar.gz ${curDir} && rm -rf ${curDir}
        fi
    done


    #rm rmDaysAgo log
    curFiles=`find . -maxdepth 1 -type f  -regex "^\.\/[0-9]+\-.*\.tar\.gz$"`
    for curFile in ${curFiles[*]}; do
        curFileDate=`date -d "${curFile:2:10}" +%s`
        fileDate=`expr $1 - ${curFileDate}`
        fileDateNum=`expr ${fileDate} / ${daySecond}`
        if [ ${fileDateNum} -gt ${rmDaysAgo} ]; then
            echo "remove file ${curFile}"
            rm ${curFile}
        fi
    done
    echo `date "+%Y-%m-%d %H:%M:%S"` clean app log end
}

cleanAppLogByMin() {
    echo `date "+%Y-%m-%d %H:%M:%S"` clean app log begin
    nowDay=$1
    nowHour=$2
    echo "step1: nowDay: ${nowDay}, nowHour: ${nowHour}."
    
    if [ ${nowHour} = "00" ]; then
       nowHour=23
       nowDay=`date -d "${nowDay} - 1 day" "+%Y-%m-%d"`
    else
       nowHour=`date -d "${nowHour} - 1 hour" "+%H"`
    fi
    echo "step2: nowDay: ${nowDay}, nowHour: ${nowHour}."
    cd ${logs}/${nowDay}
    delFile="*-${nowDay}-${nowHour}"
    for f in `ls ${delFile}`; do
        tar -zcf ${f}.tar.gz ${f} && rm ${f}    
    done
    echo `date "+%Y-%m-%d %H:%M:%S"` clean app log end
}


rmDaysAgoFile() {
    #rm rmDaysAgo log
    nowDay=$1
    cd ${logs}
    curDirs=`find . -maxdepth 1 -type d  -regex "^\.\/[0-9]+\-.*$"`
    for curDir in ${curDirs[*]}; do
        curDirDate=`date -d "${curDir:2:10}" +%s`
        dirDate=`expr ${nowDay} - ${curDirDate}`
        dirDateNum=`expr ${dirDate} / ${daySecond}`
        if [ ${dirDateNum} -ge ${rmDaysAgo} ]; then
            echo "remove dir ${curDir}"
            rm -rf ${curDir}
        fi
    done
    echo `date "+%Y-%m-%d %H:%M:%S"` remove ${rmDaysAgo} day dir end.
}


###clean magpie flow control log
cleanMagpieLog() {
    echo `date "+%Y-%m-%d %H:%M:%S"` clean magpie flow-ctrl log begin
    ###package zipDaysAgo log
    cspLogFiles=`find . -maxdepth 1 -type f -regex ".*log\.[0-9]+\-.*$"`
    for cspLogFile in ${cspLogFiles[*]}; do
        cspFileTime=`echo ${cspLogFile:2} | awk -F "." '{print $3}'`
        cspFileDate=`date -d "${cspFileTime}" +%s`
        diffCspFileDate=`expr $1 - ${cspFileDate}`
        diffCspFileDateNum=`expr ${diffCspFileDate} / ${daySecond}`
        if [ ${diffCspFileDateNum} -ge ${zipDaysAgo} ]; then
            echo "cspFileTime:$cspFileTime"
            if [ ! -d "$cspFileTime" ]; then
                mkdir -p $cspFileTime
            fi
            echo "mv file ${cspLogFile} into ${cspFileTime}"
            mv ${cspLogFile} ${cspFileTime}
        fi
    done

    #tar zipDaysAgo log
    curCspDirs=`find . -maxdepth 1 -type d  -regex "^\.\/[0-9]+\-.*$"`
    for curCspDir in ${curCspDirs[*]}; do
        curCspDirDate=`date -d "${curCspDir:2}" +%s`
        diffCsbDate=`expr $1 - ${curCspDirDate}`
        diffCsbDateNum=`expr ${diffCsbDate} / ${daySecond}`
        if [ ${diffCsbDateNum} -ge ${zipDaysAgo} ]; then
            echo "tar dir ${curCspDir}"
            tar -zcf ${curCspDir}.tar.gz ${curCspDir} && rm -rf ${curCspDir}
        fi
    done

    ###rm rmDaysAgo log
    curCspZipFiles=`find . -maxdepth 1 -type f  -regex "^\.\/[0-9]+\-.*\.tar\.gz$"`
    for curCspZipFile in ${curCspZipFiles[*]}; do
        curCspZipFileDate=`date -d "${curCspZipFile:2:10}" +%s`
        cspZipFileDate=`expr ${todayDate} - ${curCspZipFileDate}`
        cspZipDateNum=`expr ${cspZipFileDate} / ${daySecond}`
        if [ ${cspZipDateNum} -ge ${rmDaysAgo} ]; then
            echo "remove file ${curCspZipFile}"
            rm ${curCspZipFile}
        fi
    done
    echo `date "+%Y-%m-%d %H:%M:%S"` clean magpie flow-ctrl log end
}

######################################################################

##目前日志切割在45分完成，所以设定在50分之后进行清理
nowMin=`date "+%M"`
##到第一次执行时间隔时间
intlMin=`expr ${execMin} - ${nowMin}`
if [ ${intlMin} -ge 0 ]; then
    echo `date "+%Y-%m-%d %H:%M:%S"` "now minute is ${nowMin}, will sleep ${intlMin} minutes."
    intlSec=`expr ${intlMin} \* 60`
    sleep ${intlSec}
fi

while true
do
    ###清理应用日志
    curHour=`date "+%H"`
    today=`date "+%Y-%m-%d"`
    todayDate=`date -d "${today}" +%s`
    if [ -d "$logs" ] ; then
        cd $logs
        cleanAppLogByMin $today $curHour
    else
        echo "$logs not exists, no need clean."
    fi 
    
    ###清理备份日志及magpie日志
    if [ $curHour == $execTime ]; then
        ####clean app log
        if [ -d "$logs" ] ; then
            cd $logs
            rmDaysAgoFile $todayDate
        else
            echo "$logs not exists, no need remove."
        fi

        ###clean magpie flow control log
        if [ -d "$cspLogs" ] ; then
            cd $cspLogs
            cleanMagpieLog $todayDate
        else
            echo "$cspLogs not exists, no need clean."
        fi
    else
        echo "Now time $curHour, not at exec time $execTime, no need clean magpie log!"
    fi
    
    ###根据下一次执行的时间计算休眠时间
    curMinAfterExec=`date "+%M"`
    nextIntlMin=`expr \( ${execMin} - ${curMinAfterExec} \) + 60`
    nextIntlSec=`expr ${nextIntlMin} \* 60`
    echo `date "+%Y-%m-%d %H:%M:%S"`", will sleep ${nextIntlSec} minutes until next exec."

    sleep ${nextIntlSec}
done

备注：78-81行验证：

[cqpsrvas@spchd02qyx86 logs]$curDirs=`find . -maxdepth 1 -type d  -regex "^\.\/[0-9]+\-.*$"`   
[cqpsrvas@spchd02qyx86 logs]$echo $curDirs
./2021-06-01 ./2021-06-03 ./2021-06-02 ./2021-06-05
[cqpsrvas@spchd02qyx86 logs]$for curDir in ${curDirs[*]}; do
> curDirDate=`date -d "${curDir:2:10}" +%s`
> echo $curDirDate 
> done
1622476800
1622649600
1622563200
1622822400

2. 获取进程 PID

下面脚本可以获取名称中包含java关键字的进程ID，由于执行该命令同时也会产生一个包含--color=auto关键字的进程，所以还需要继续使用grep -v color'，以排除该进程：

ps -ef | grep $USER | grep java | grep -v color | awk '{print $2}'

关于awk的简单使用可见使用awk命令获取文本的某一行，某一列。

3. 优雅启停 upjas

4. 日志统计脚本

4.1 统计日志类型和数量

4.1.1 需求

在首页常应用个性化推荐系统2021年5月17日的 ads_fpr_app_error.log 日志中，统计22:56至22:58分的日志种类和数量。

一条典型的日志如下所示：

2021-05-17 22:56:54 720|COMMON_APP_AREA_THREAD--34|WARN ||ADS021051722565400000000024636|AsyncTask| # CheckGetAppCallBack[cqp_base_appgroup_query] The flow is already terminated.

4.1.2 脚本

下列脚本语句的重点在于：先排序再去重。

grep -i '22:5[6-8]' ads_fpr_app_error.log-2021-05-17-22 | awk '{print $6}' | sort | uniq -c

5. 生成随机字符串

5.1 需求

生成一个长度为8个字符的字符串，且只能包含大小写字母或-和_这两个特殊字符。

5.2 脚本

#!/bin/bash

# Declare the variable denoting alphabet
ALPHABET="abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_-"

# Define the function for generating a random int within a given range
function rand(){
    min=$1
    max=$(($2-$min+1))
    num=$(date +%s%N)
    echo $(($num%$max+$min))
}

# Declare the variable for controlling the loop times
length=1
# Decalre the variable for storing the desired string
string=""
# Loop length times to get the desired string
while [ $length -le 8 ]
do
    # Generate a random int
    rand_num=$(rand 0 53)
    # Using $rand_num as an index
    char=${ALPHABET:$rand_num:1}
    # Concatenate chars into string
    string=$string$char
    let length++
done

echo $string

exit 0

实际上，Linux生成随机数这篇文章中有现成的轮子，使用mkpasswd -l 8 -s 0即可以实现类似的效果，需要注意的是，如果Linux发行版中未安装该命令对应的软件包，则需要先使用yum install -y expect命令进行安装。

6. 指定时间执行操作

while :
do
        curHour=`date +%H`
        curMin=`date +%M`
        if [[ 10#$curHour -eq 20 && 10#$curMin -eq 00 ]] ; then
                echo $curHour:$curMin # 仅在20:00的时候打印时间
        sleep 60
        fi
done

在 shell 脚本中， -eq 一般用于数字之间是否相等的比较， = 一般用于字符串之间是否相等的比较。

7. shell 脚本查询数据库

#!/bin/bash
host=$ip_of_proxy
port=$port
user=$db_username
dbname=$dbname
# 需修改为生产路径
DATA_HOME=$path

# read -s -p "Enter current password for $user: " pwd
pwd=$passwd

current_day=`date +%F`
current_day="${current_day} 00:00:00"
last_day=`date -d last-day +%F`
last_day="${last_day} 00:00:00"
current_day_f=`date +%Y%m%d`
last_day_f=`date -d last-day +%Y%m%d`

echo ''
echo "start to cal result of ${last_day_f}"


#每日报名的低贡献用户
newAddUser=0
for ((i=0; i<=9; i++))
do
	tableNameLow=tbl_mac_market_usr_info_${i}
	countLowSql="SELECT count(*) FROM ${tableNameLow} WHERE crt_ts>='${last_day}' and crt_ts<='${current_day}' and user_level='0'"
	resultLow=`mysql -h$host -P$port -u$user -p$pwd $dbname -N -e "${countLowSql}"`
	newAddUser=`expr $newAddUser + $resultLow`

done
echo "${last_day_f} 低贡献用户总量：${newAddUser}" >> ./result/usrInfoDailyResult.txt



#每日新增的高贡献用户
newLevelUser=0
for ((i=0; i<=9; i++))
do
	tableNameHigh=tbl_mac_market_usr_info_${i}
	countHighSql="SELECT count(*) FROM ${tableNameHigh} WHERE user_level='1' and upd_ts>='${current_day}'"
	resultHigh=`mysql -h$host -P$port -u$user -p$pwd $dbname -N -e "${countHighSql}"`
	newLevelUser=`expr $newLevelUser + $resultHigh`
done
echo "${last_day_f} 升级贡献用户总量：${newLevelUser}" >> ./result/usrInfoDailyResult.txt

#每日发生交易的低贡献用户数
activedUser=0
for ((i=0; i<=9; i++))
do
	userTableNameLow=tbl_mac_market_usr_info_${i}
	trandDetailTableNameLow=tbl_mac_trans_detail_${i}
	
	#mysql -h10.200.56.17 -P51879 -umac_db -pOStem@00 cqpmacdb -e "" -N
	countLowSql="select count(TBLTMP.ttid) from (select distinct(user_id) as ttid from ${trandDetailTableNameLow} where crt_ts>='${current_day}' and user_id in(select user_id from ${userTableNameLow} where user_level='0')) as TBLTMP"
	resultLow=`mysql -h$host -P$port -u$user -p$pwd $dbname -N -e "${countLowSql}"`
	activedUser=`expr $activedUser + $resultLow`
done
# 补充已升级用户
activedUser=`expr $activedUser + $newLevelUser`

echo "${last_day_f} 每日发生交易的低贡献用户数 ：${activedUser}" >> ./result/usrInfoDailyResult.txt


#每日用户享有券包抽取资格
userQual=0
file_names=$(ls ${DATA_HOME}/daydraw/${current_day_f})
for file_name in ${file_names}
do
	if [[ ${file_name:0:1} != "e" ]]
	then
		size=`cat ${DATA_HOME}/daydraw/${current_day_f}/$file_name | wc -l`
		userQual=`expr ${userQual} + ${size}`
	fi
done
echo "${last_day_f} 拥有抽奖资格用户总量：${userQual}" >> ./result/usrInfoDailyResult.txt


#每日低贡献用户做了多少笔交易
transCount=0
file_names=$(ls ${DATA_HOME}/lowusrmisdetails/${current_day_f})
for file_name in ${file_names}
do
	if [[ ${file_name:0:1} != "e" ]]
	then
		size=`cat ${DATA_HOME}/lowusrmisdetails/${current_day_f}/$file_name | wc -l`
		transCount=`expr ${transCount} + ${size}`
	fi
done
echo "${last_day_f} 低贡献用户总交易笔数：${transCount}" >> ./result/usrInfoDailyResult.txt