shell脚本定期清理elasticsearch日志

前景：部署skywalking，elk等服务时会用到es存储数据，运行一段时间会占用磁盘空间，影响服务器性能。在保证服务正常使用的情况下，我们可以对es的索引进行删除操作，舍弃旧数据。所以我编写了一个脚本，仅供参考。如果要做定时任务的话，可以对稍作修改。

[root@dev-sw_es:shell]# vim delete_es_data.sh 
#!/bin/bash
#本脚本用于按月份清理ES存储数据
#获取本月份与索引中的日期进行比较，本月份前的索引数据将被删除。
#删除前要确认
 
NOW_DATE=`date +%Y%m`
ALLLINES=`/usr/bin/curl -s -XGET http://127.0.0.1:9200/_cat/indices?v | egrep *-20*`
 
echo
echo "THIS IS WHAT SHOULD BE DELETED FOR ES:"
echo
 
echo "${ALLLINES}" | while read LINE
do
  FORMATEDLINE=`echo ${LINE} |awk '{ print $3 }' | awk -F'-' '{ print $2 }' | cut -c 1-6 `
  #echo ${FORMATEDLINE}
  if [ -n "${FORMATEDLINE}" -a "${FORMATEDLINE}" -lt "${NOW_DATE}" ]
  then
    TODELETE=`echo ${LINE} | awk '{ print $3 }'`
    echo "http://127.0.0.1:9200/${TODELETE}"
  fi
done
 
echo
echo -n "if this make sence, Y to continue N to exit [Y/N]:"
read INPUT
if [ "${INPUT}" == "Y" ] || [ "${INPUT}" == "y" ] || [ "${INPUT}" == "yes" ] || [ "${INPUT}" == "YES" ]
then
  echo "${ALLLINES}" | while read LINE
  do
    FORMATEDLINE=`echo ${LINE} | awk '{ print $3 }' | awk -F'-' '{ print $2 }' | cut -c 1-6 `
    if [ -n "${FORMATEDLINE}" -a "${FORMATEDLINE}" -lt "${NOW_DATE}" ]
    then
      TODELETE=`echo ${LINE} | awk '{ print $3 }'`
      /usr/bin/curl -XDELETE http://127.0.0.1:9200/$TODELETE
      sleep 1
      fi
  done
else
  echo SCRIPT CLOSED BY USER, BYE ...
  echo
  exit
fi
~  

写进crontab的脚本可以参考：

#!/bin/bash
 
NOW_DATE=`date +%Y%m`
#NOW_DATE=202010
ALLLINES=`/usr/bin/curl -s -XGET http://127.0.0.1:9200/_cat/indices?v | egrep *-20*`
 
echo "${ALLLINES}" | while read LINE
do
  FORMATEDLINE=`echo ${LINE} |awk '{ print $3 }' | awk -F'-' '{ print $2 }' | cut -c 1-6 `
  if [ -n "${FORMATEDLINE}" -a "${FORMATEDLINE}" -lt "${NOW_DATE}" ]
  then
    TODELETE=`echo ${LINE} | awk '{ print $3 }'`
    #echo "http://127.0.0.1:9200/${TODELETE}"
    /usr/bin/curl -XDELETE http://127.0.0.1:9200/$TODELETE
    sleep 1
  fi

ELK的索引删除：

#I use a bash script, just change the 30 with the # of days you want to keep
 
#!/bin/bash
 
# Zero padded days using %d instead of %e
DAYSAGO=`date --date="30 days ago" +%Y%m%d`
ALLLINES=`/usr/bin/curl -s -XGET http://127.0.0.1:9200/_cat/indices?v | egrep logstash`
 
echo
echo "THIS IS WHAT SHOULD BE DELETED FOR ELK:"
echo
 
echo "$ALLLINES" | while read LINE
do
  FORMATEDLINE=`echo $LINE | awk '{ print $3 }' | awk -F'-' '{ print $2 }' | sed 's/\.//g' ` 
  if [ "$FORMATEDLINE" -lt "$DAYSAGO" ]
  then
    TODELETE=`echo $LINE | awk '{ print $3 }'`
    echo "http://127.0.0.1:9200/$TODELETE"
  fi
done
 
echo
echo -n "if this make sence, Y to continue N to exit [Y/N]:"
read INPUT
if [ "$INPUT" == "Y" ] || [ "$INPUT" == "y" ] || [ "$INPUT" == "yes" ] || [ "$INPUT" == "YES" ]
then
  echo "$ALLLINES" | while read LINE
  do
    FORMATEDLINE=`echo $LINE | awk '{ print $3 }' | awk -F'-' '{ print $2 }' | sed 's/\.//g' `
    if [ "$FORMATEDLINE" -lt "$DAYSAGO" ]
    then
      TODELETE=`echo $LINE | awk '{ print $3 }'`
      /usr/bin/curl -XDELETE http://127.0.0.1:9200/$TODELETE
      sleep 1
      fi
  done
else 
  echo SCRIPT CLOSED BY USER, BYE ...
  echo
  exit
fi