使用groovy脚本自动清理过多Elasticsearch索引

最新推荐文章于 2024-02-17 10:16:11 发布

Just2Do_

最新推荐文章于 2024-02-17 10:16:11 发布

阅读量1.4k

点赞数

本文链接：https://blog.csdn.net/candyzh/article/details/62223601

版权

之前搭建了一套ELK方案分布日志采集系统，一直运行比较稳定，每日采集日志量600万+，存放es索引的磁盘只有150G，一般1个月左右就满了，所以必须定期清理一下旧索引，以免爆了引起连锁反应，导致整个应用链都挂了。

手工清晰比较麻烦，花了点时间，使用groovy写了个脚本，配合linux cron定时每天执行，效果理想。

@Grab('com.squareup.okhttp:okhttp:2.5.0')
@Grab('org.slf4j:slf4j-log4j12:1.7.12')


import com.squareup.okhttp.Callback;
import com.squareup.okhttp.OkHttpClient;
import com.squareup.okhttp.Request;
import com.squareup.okhttp.Response;

import org.apache.log4j.Logger;
import org.apache.log4j.PropertyConfigurator;


Properties pro = new Properties();
pro.put("log4j.rootLogger", "DEBUG,file"); 
pro.put("log4j.appender.file", "org.apache.log4j.RollingFileAppender"); 
pro.put("log4j.appender.file.File", "/home/apps/mamaner_index_shell/logs/info.txt"); 
pro.put("log4j.appender.file.Threshold", "info");
pro.put("log4j.appender.file.MaxFileSize", "1024kb");
pro.put("log4j.appender.file.MaxBackupIndex", "100");
pro.put("log4j.appender.file.encoding", "UTF-8");
pro.put("log4j.appender.file.layout", "org.apache.log4j.PatternLayout");
pro.put("log4j.appender.file.layout.ConversionPattern", "[%d] [%-5p] [%c.%M:%L] - %m%n");
PropertyConfigurator.configure(pro);

Logger logger = Logger.getLogger(this.class);
logger.info("Start to check index....");

//获取某个区间磁盘剩余百分比
def getDiskFreePercent = {
        String info -> 
        def a = info.split("%")[0]
        def p = a.substring(a.lastIndexOf(" ")).trim()
}
//获取整个磁盘的信息
def getDiskinfo=
{
        try {
            Process process = Runtime.getRuntime().exec("df -m");
            process.waitFor();
            InputStreamReader ir = new InputStreamReader(process.getInputStream());
            LineNumberReader input = new LineNumberReader(ir);
            
            String line;
            while ((line = input.readLine()) != null) {
             if (line.indexOf("/home")>=0)
                return line;
            }
        } catch (java.io.IOException e) {
            System.err.println("IOException " + e.getMessage());
        }
}
//http删除
def httpDelete={
    String url->
    
    logger.info("DELETE URL:"+url);

    OkHttpClient client = new OkHttpClient();
    Request request = new Request.Builder().url(url).delete(null).build();
    Response response = client.newCall(request).execute();  
    if (response.isSuccessful()) {     
       println response.body().string();
    }
    else {  
          println response;
    }
}

//获取当前索引目录并排序
def getIndexFolders={

    def indexFoldList = new ArrayList<String>();
    def File targetFile = new File("/home/apps/log-watcher/elasticsearch-2.1.1/data/elasticsearch/nodes/0/indices")
    targetFile.eachDir {
        File indFold->
        if (indFold.getName().indexOf("logstash")>=0) indexFoldList.add(indFold.getName())
    }
    Collections.sort(indexFoldList,new Comparator<String>(){
                public int compare(String arg0, String arg1) {
                    return arg0.compareTo(arg1);
                }
            });
    return indexFoldList;
}

//超过预定磁盘占用，开始从最早日期删除索引，知道容量满足
def captain = 70;
for(;getDiskFreePercent(getDiskinfo()).toFloat()>captain;)
{
     httpDelete("http://localhost:9200/"+getIndexFolders()[0])
}

logger.info("Finish checked index!");