1 简介
本方案实现的功能点:输入日志文件、按照指定的日期(天数)清理数据、按照指定的磁盘路径使用率阈值清理数据、清除指定的索引数据,并配合crond实现定时监控和清理索引数据,以确保磁盘空间健康。
2 使用方法
脚本名称为:clearIndex.sh
2.1 重要变量解释
DISK_PATH='/home'
解释:监控的磁盘路径,此处监控/home路径的使用率
DISK_THRESHOLD=70
解释:监控的磁盘路径的使用率百分比,此处是监控的磁盘路径使用率超过70%就符合启动清理
TARGET_DATE_NUMBER=180
解释:清理指定天数之前对应日期的数据,此处是清理的是180天前对应日期那天的所有数据
2.1 检查磁盘路径使用率并清理数据
./clearIndex.sh start_for_diskThreshold
2.2 清理指定天数前日期的数据
./clearIndex.sh start_for_specifiedDate
2.3 清理指定索引数据
./clearIndex.sh start_for_index test.helloword-20220321
2.4 自动监控并清理数据
配合crond任务管理器,定时执行2.1,2.2即可实现自动监控和自动清理,确保磁盘空间健康。
#!/bin/bash
#Usage: Automatic clearing the indexes of ES!
#Maintenance: xxxxx
#Description: Here are offered two method of deletion indexes.
#Method 1: Specify the date that you expect to clear the indexes.
#Method 1: Specify the server disk space usage threshold for clearing indexes.
#
CURDIR="$( cd "$( dirname "$0" )" && pwd )"
#Log file path
LOGPATH=$CURDIR/logs
if [[ ! -d $LOGPATH ]];then
mkdir $LOGPATH
fi
#################################################################################################################
#
#Configuration:
#The user/password of es
ES_USER='xxxxxx'
ES_PASS='xxxxxx'
#The disk space path
DISK_PATH='/home'
DISK_THRESHOLD=70
CHECK_DISK=`df -h | grep $DISK_PATH | tr '%' ' ' | awk '$5>'$DISK_THRESHOLD'{print $5}' | wc -l`
#The url of es
ES_URL='https://localhost:9200/'
#Specify the prefix of index needing to delete.
PREFIX=test
#Specify the date of deletion indexes by many different date formats
#exp.: 1 month ago ; +%Y-%m-%d
TARGET_DATE_NUMBER=180
#
#################################################################################################################
#TARGET_DATE=`date -d "3 week ago" +%Y%m%d`
#TARGET_DATE=`date -d "6 month ago" +%Y%m%d`
TARGET_DATE=`date -d "$TARGET_DATE_NUMBER day ago" +%Y%m%d`
#The current date
#CURRENT_DATE=`date +%Y%m%d--%T`
CURRENT_DATE=`date +%Y%m%d`
#Query the all indexes
#curl -k --basic -u ${ES_USER}:${ES_PASS} ${ES_URL}_cat/indices?v
#Print the log information and output it into a file.
display(){
local type=$1
local msg=${@:2}
curDateTime=`date +'%Y-%m-%d %H:%M:%S.%N'`
if [[ $type = "[Info]" ]]; then
echo -e "\\033[1;36;40m[Info] ${curDateTime} $msg \\033[0m"
elif [[ $type = "[Error]" ]]; then
echo -e "\\033[1;31;40m[Error] ${curDateTime} $msg \\033[0m"
elif [[ $type = "[Exec]" ]]; then
echo -e "\\033[1;33;40m[Exec] ${curDateTime} $msg \\033[0m"
elif [[ $type = "[Success]" ]]; then
echo -e "\\033[1;32;40m[Success] ${curDateTime} $msg \\033[0m"
else
echo -e "${curDateTime} $@"
fi
echo "${curDateTime} $type $msg " >> ${LOGPATH}/${USER}-indexClear-${dateNYR}.log
}
#To countdown
function countdownTimer(){
#countdown
time=$1
for((i=time;i>0;i--))
do
display [Exec] "${i} seconds to start:"
sleep 1
done
}
#Clear the specified index
function clear_index(){
specifiedIndex=$1
if [[ ! -n "$specifiedIndex" ]];then
display [Error] "The specified index to be deleted mustn't be null!"
return
fi
curl -k --basic -u ${ES_USER}:${ES_PASS} -XDELETE ${ES_URL}${specifiedIndex}
if [ $? -eq 0 ];then
display [Success] "-->${CURRENT_DATE} Clear '${specifiedIndex}' success.."
else
display [Error] "-->${CURRENT_DATE} Clear '${specifiedIndex}' failed.."
fi
}
#Clear the indexes by specified date
function clear_indexes(){
specifiedPrefix=$1
specifiedDate=$2
display [Info] "Query the target indexes '${specifiedPrefix}.*${specifiedDate}' ready to clear!"
#Query the specified indexes by the date format
#curl -k --basic -u ${ES_USER}:${ES_PASS} -XGET ${ES_URL}{$PREFIX}.*${TARGET_DATE}
TARGET_INDEXES=`curl -k --basic -u ${ES_USER}:${ES_PASS} -XGET ${ES_URL}_cat/indices?v | awk '{print $3}' | grep "${specifiedPrefix}.*${specifiedDate}"`
for index in ${TARGET_INDEXES};do
#Delete the indexes specifying date format
#curl -k --basic -u ${ES_USER}:${ES_PASS} -XDELETE ${ES_URL}{$PREFIX}.*${TARGET_DATE}
curl -k --basic -u ${ES_USER}:${ES_PASS} -XDELETE ${ES_URL}${index}
if [ $? -eq 0 ];then
display [Success] "-->${CURRENT_DATE} Clear '${index}' success.."
else
display [Error] "-->${CURRENT_DATE} Clear '${index}' failed.."
fi
done
}
#Start to clear the indexes by the disk threshold.
function start_for_diskThreshold(){
if [ $CHECK_DISK -gt 0 ];then
#checked overload
currentDiskUsage=`df -h | grep $DISK_PATH | awk '{print $5}'`
display [Exec] "Warning! The current disk usage is $currentDiskUsage (>=${DISK_THRESHOLD}% ) , the disk is overload! It will be clearing the indexes!"
clear_indexes $PREFIX $TARGET_DATE
display [Info] "This round to clear the indexes is completed,continuing to go next check round!"
countdownTimer 10
#Modify the 'TARGET_DATE' to add one day backward.
TARGET_DATE_NUMBER=$((TARGET_DATE_NUMBER - 1))
TARGET_DATE=`date -d "$TARGET_DATE_NUMBER day ago" +%Y%m%d`
display [Exec] "TARGET_DATE_NUMBER is $TARGET_DATE_NUMBER! TARGET_DATE is $TARGET_DATE"
main
else
#checked normal
display [Success] "The disk is nomal! Everything is ok, today is a good day! The round to check is completed!"
fi
}
#Start to clear the indexes by the prefix and the date
function start_for_specifiedDate(){
clear_indexes $PREFIX $TARGET_DATE
}
#Start to clear the index by the index.
function start_for_index(){
clear_index $1
}
case $1 in
byDate)
start_for_specifiedDate
;;
byDisk)
start_for_diskThreshold
;;
byIndex)
start_for_index $2
;;
*)
display [Info] "Usage: {byDate|byDisk|byIndex index}"
;;
esac