JAVA定时器每日删除ES过期索引

JAVA定时器每日删除ES过期索引

背景：在Linux服务器上，设置一个后台进程，每天0点10分自动删除ELK中Elasticsearch中超过三个月的旧索引，使用Quartz实现。

废话少说，直接上代码！！！

目录：

1、源码：

2、调用方法：

1、源码：

（1）QuartzCli.java

package com.remoa.quartzCli;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.quartz.*;
import org.quartz.impl.StdSchedulerFactory;

/**
 * This class is used to use Quartz
 * @author remoa
 * @since 2017.09.18
 */
public class QuartzCli{
    //日志记录
    private static final Log LOG = LogFactory.getLog(QuartzCli.class);
    //后台流程每天0点10分开始跑
    //public static final String INDEX_DELETE_CRON = "0 10 0 * * ?";
    //后台流程每10秒开始跑
    private static final String INDEX_DELETE_CRON = "*/10 * * * * ?";
    //定义Job名字
    private static final String JOB_NAME = "Index_quartz_cli";

    /**
     * 启动定时任务
     */
    public void start(){
        System.out.println("Start quartz mode: " + INDEX_DELETE_CRON);
        LOG.info("Start quartz mode: " + INDEX_DELETE_CRON);
        //SchedulerFactory：提供用于获取调度程序实例的客户端可用句柄的机制
        SchedulerFactory schedulerFactory = new StdSchedulerFactory();
        //Scheduler：调度程序，接口，代表一个独立运行容器，调度程序维护JobDetail和触发器的注册表，一旦注册，调度程序负责执行作业。
        Scheduler scheduler = null;
        try {
            //设置调度程序的实现类
            scheduler = schedulerFactory.getScheduler();
        } catch (SchedulerException e) {
            LOG.error(e.getMessage(), e);
        }
        //实例化JobDetail调度任务，JobDetail存放作业的状态，JobDetail对象储存作业的侦听器、群组、数据映射、描述以及作业的其它属性。
        //newJob方法设置需要执行的Job任务类名
        JobBuilder jobBuilder = JobBuilder.newJob(MyJob.class);
        //withIdentity设置JobDetail的name和group
        jobBuilder.withIdentity(JOB_NAME, "remoaGroup");
        //JobDetail是一个接口，必须通过JobBuilder的build方法实例化JobDetail
        JobDetail job = jobBuilder.build();
        //实例化Trigger触发器，实现对作业的调度。
        CronTrigger cronTrigger = (CronTrigger)TriggerBuilder.newTrigger()
                .withIdentity("Cron_Trigger" + System.currentTimeMillis(), "remoaGroup")
                .withSchedule(CronScheduleBuilder.cronSchedule(INDEX_DELETE_CRON))
                .startNow()
                .build();
        //JobDataMap是java map的具体实现，并添加了一些便利的方法用于存储与读取原生类型数据，可以为Job实例提供属性/配置
        job.getJobDataMap()
                .put("executor", this);
        try {
            //为调度程序设置job和trigger
            scheduler.scheduleJob(job, cronTrigger);
            //调用start开始调度
            scheduler.start();
        } catch (Exception e) {
            LOG.error(e.getMessage(), e);
        }
    }

    public static void main(String[] args) {
        QuartzCli quartzCli = new QuartzCli();
        quartzCli.start();
    }

}

（2）MyJob.java

package com.remoa.quartzCli;

import com.remoa.index.IndexProcess;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.quartz.Job;
import org.quartz.JobExecutionContext;
import org.quartz.JobExecutionException;

import java.io.IOException;
import java.io.InputStream;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.List;
import java.util.Properties;

/**
 * This class is used to define job
 * @author remoa
 * @since 2017.09.18
 */
public class MyJob implements Job {
    private static final Log LOG = LogFactory.getLog(MyJob.class);

    //在Quartz中，所有的任务都必须实现Job接口，Job中只有一个execute(JobExecutionContext jobExecutionContext)实现方法，具体业务实现需要写在这个方法里，这个方法将会在触发器满足调度条件时触发。
    @Override
    public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException {
        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss");
        System.out.println(sdf.format(new Date()) + "------------- start job");
        LOG.info(sdf.format(new Date()) + "------------- start job");
        InputStream input = MyJob.class.getClassLoader().getResourceAsStream("indexDelete.properties");
        Properties properties = new Properties();
        try {
            properties.load(input);
        } catch (IOException e) {
            LOG.error(e.getMessage(), e);
        }
        IndexProcess indexProcess = null;
        try {
            indexProcess = new IndexProcess(properties.getProperty("keystorePath"), properties.getProperty("truststorePath"), properties.getProperty("keypassword"), properties.getProperty("trustpassword"), properties.getProperty("elasticsearchHost"), properties.getProperty("elasticsearchClusterName"));
        } catch (Throwable throwable) {
            LOG.error(throwable.getMessage());
            throwable.printStackTrace();
        }
        System.out.println("get index-------------");
        List<String> list = indexProcess.getIndex();
        System.out.println("Total number of lists :" + list.size());
        LOG.info("Total number of lists :" + list.size());
        String keyword = indexProcess.getKeyword();
        indexProcess.deleteOldIndex(list, keyword);
    }

}

（3）IndexProcess.java

package com.remoa.index;

import com.floragunn.searchguard.ssl.SearchGuardSSLPlugin;
import com.floragunn.searchguard.ssl.util.SSLConfigConstants;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
import org.elasticsearch.client.AdminClient;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.elasticsearch.transport.client.PreBuiltTransportClient;

import java.net.InetAddress;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Iterator;
import java.util.List;

/**
 * This class is used to delete old index more than 100 days in elasticsearch.
 * @author remoa
 * @since 2017.09.18
 */
public class IndexProcess {
    private TransportClient client;
    private AdminClient adminClient;
    public static final int DATE_NUMBER = -90;

    /**
     * 构造方法，提供连接Elasticsearch的相关参数并使用传输客户端TransportClient连接到Elasticsearch集群
     * @param keystorePath KeyStore签名证书位置
     * @param truststorePath Truststore签名证书位置
     * @param keypwd KeyStore签名证书密码
     * @param trustpwd Truststore签名证书密码
     * @param esHost Elasticsearch集群服务器
     * @param esClusterName Elasticsearch集群名称
     * @throws Throwable 异常不单独处理了，直接在方法中抛出
     */
    public IndexProcess(String keystorePath, String truststorePath, String keypwd, String trustpwd, String
            esHost, String esClusterName) throws Throwable{
        Settings settings = Settings.builder()
                //ES集群名称
                .put("cluster.name", esClusterName)
                .put("searchguard.ssl.transport.enabled", true)
                //私钥证书位置
                .put(SSLConfigConstants.SEARCHGUARD_SSL_TRANSPORT_KEYSTORE_FILEPATH, keystorePath)
                //公钥证书位置
                .put(SSLConfigConstants.SEARCHGUARD_SSL_TRANSPORT_TRUSTSTORE_FILEPATH, truststorePath)
                //keystore一般保存我们的私钥，用来加密解密或者为别人做签名。
                .put(SSLConfigConstants.SEARCHGUARD_SSL_TRANSPORT_KEYSTORE_PASSWORD, keypwd)
                //truststore里存放的是只包含公钥的数字证书。
                .put(SSLConfigConstants.SEARCHGUARD_SSL_TRANSPORT_TRUSTSTORE_PASSWORD, trustpwd)
                .put(SSLConfigConstants.SEARCHGUARD_SSL_TRANSPORT_ENFORCE_HOSTNAME_VERIFICATION, false)
                .put("path.home", ".")
                //使得客户端去嗅探整个集群的状态，把集群中其它机器的IP地址加到客户端中，设置为true则不用手动设置集群里所有机器的IP连接到客户端，它会自动帮助添加，并自动发现新加入集群的机器。
                .put("client.transport.sniff", true)
                .build();
        client = new PreBuiltTransportClient(settings, SearchGuardSSLPlugin.class)
                .addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(esHost), 9301))
                .addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(esHost), 9300));
        adminClient = client.admin();
    }

    /**
     * 根据索引名删除索引
     * @param index 索引名
     */
    public void deleteIndex(String index){
        adminClient.indices().
                delete(new DeleteIndexRequest(index)).
                actionGet();
    }

    /**
     * 得到全部索引
     * @return 返回List封装的全部索引的名字
     */
    public List<String> getIndex(){
        List<String> list = new ArrayList<String>();
        String [] indices = adminClient.indices()
                .prepareGetIndex()
                .setFeatures()
                .get()
                .getIndices();
        for(int i = 0; i < indices.length; i++){
            list.add(indices[i]);
        }
        return list;
    }

    /**
     * 得到过滤的关键字，即xxxx.xx.xx(年月日字符串)，这里设置为90天
     * @return 过滤关键字
     */
    public String getKeyword(){
        Calendar calendar = Calendar.getInstance();
        calendar.add(Calendar.DATE, DATE_NUMBER);
        String year = String.valueOf(calendar.get(Calendar.YEAR));
        String month = String.valueOf(calendar.get(Calendar.MONTH) + 1);
        String day = String.valueOf(calendar.get(Calendar.DATE));
        if(month.length() == 1){
            month = "0" + month;
        }
        if(day.length() == 1){
            day = "0" + day;
        }
        String keyword = year + "." + month + "." + day;
        return keyword;
    }

    /**
     * 删除所有的超期的旧索引
     * @param list 索引列表
     * @param keyword 过滤关键字
     */
    public void deleteOldIndex(List<String> list, String keyword){
        int count = 0;
        for(Iterator<String> iter = list.iterator(); iter.hasNext(); ){
            String str = iter.next();
            //索引名称字段长度比10小的话就不作为删除对象了
            if(str.length() > 10){
                String dateStr = str.substring(str.length() - 10 ,str.length());//取出索引名称字段的结尾字段
                //索引名称字段最后不是以日期结尾的话也不作为删除对象
                String patternStr = "^[0-9]{4}\\.[0-9]{2}\\.[0-9]{2}$";
                if(dateStr.matches(patternStr)){
                    if(str.substring(str.length() - 10 ,str.length()).compareTo(keyword) < 0){
                        System.out.println("index： " + str + " is deleted");
                        //this.deleteIndex(str);
                        count++;
                    }
                }
            }
        }
        System.out.println("The total delete index is: " + count);
    }

}

（4）log4j2.xml

<?xml version="1.0" encoding="UTF-8" ?>
<!-- monitorInterval：自动检测修改配置文件和重新配置本身，设置间隔秒数-->
<Configuration stauts="WARN" monitorInterval="60">
    <!-- Appender附加器对象，负责输出日志信息到不同的地方，如数据库，文件，控制台等等-->
    <Appenders>
        <Console name="Console" target="SYSTEM_ERR">
            <!--
                输出日志的格式
                %-5p：用空格右垫，当事件级别的名称少于五个字符
                %F：用于输出被发出日志记录请求的文件名，带.java后缀及包名称
                %L：用于输出从被发出日志记录请求的行号
                %m：输出记录事件的消息内容
                %x：用于与产生该日志事件的线程相关联输出的NDC
                %n：输出行分隔符或文字
                %t：用于输出生成的日志记录事件的线程的名称，比如在main方法中，将输出main
            -->
            <PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss} [%-5p] --> %F(%L): %m %x %n" />
        </Console>
        <!-- 每次大小超过size，则size大小的日志会自动存入按年份-月份建立的文件夹下面并进行压缩，作为存档。 -->
        <RollingFile name="RollingFileInfo" fileName="/root/remoa/logs/deleteIndex.log" filePattern="/root/remoa/logs/$${date:yyyy-MM}/deleteIndex-%d{yyyy-MM-dd}-%i.log">
            <!-- 只输出level及以上级别的事件，其它的直接拒绝 -->
            <ThresholdFilter level="warn" onMatch="ACCEPT" onMismatch="DENY" />
            <PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss,SSS} [%C{2}] [%p %L] [%t] %m [ %x] %n" />
            <Policies>
                <TimeBasedTriggerPolicy/>
                <SizeBasedTriggerPolicy size="100MB" />
            </Policies>
            <!-- 设置默认同一个文件夹下文件数目，默认为7个，这里设置为10个 -->
            <DefaultRolloverStrategy max="10" />
        </RollingFile>
    </Appenders>
    <!-- 定义记录器对象，Logger对象负责捕获日志信息及它们存储在一个空间的层次结构 -->
    <Loggers>
        <!-- 全局的级别为info，可通过具体appender中的ThresholdFilter进行设置别的级别 -->
        <Root level="info">
            <!-- 只有引入了logger并引入的appender，appender才会生效 -->
            <AppenderRef ref="Console" />
            <AppenderRef ref="RollingFileInfo" />
        </Root>
    </Loggers>
</Configuration>

（5）pom.xml

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>
    <groupId>com.remoa</groupId>
    <artifactId>indexTest</artifactId>
    <version>1.0-SNAPSHOT</version>
    <build>
        <finalName>indexTest</finalName>
        <!-- 默认源代码和资源文件目录配置 -->
        <sourceDirectory>src/main/java </sourceDirectory>
        <testSourceDirectory>src/test/java</testSourceDirectory>
        <resources>
            <resource>
                <directory>src/main/resources</directory>
            </resource>
        </resources>
        <testResources>
            <testResource>
                <directory>src/test/resources</directory>
            </testResource>
        </testResources>
        <plugins>
            <!-- 编译插件，处理maven项目管理因为版本不一致导致编译不通过的问题 -->
            <plugin>
                <artifactId>maven-compiler-plugin</artifactId>
                <version>3.3</version>
                <configuration>
                    <!-- 源代码使用的开发版本 -->
                    <source>1.8</source>
                    <!-- 需要生成的目标class文件的编译版本 -->
                    <target>1.8</target>
                </configuration>
            </plugin>
            <plugin>
                <!-- 将依赖的某个jar包内部的类或者资源include/exclude掉-->
                <!-- 通过maven-shade-plugin插件生成一个uber-jar，它包含所有的依赖jar包-->
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-shade-plugin</artifactId>
                <version>2.4.3</version>
                <executions>
                    <execution>
                        <phase>package</phase>
                        <goals>
                            <goal>shade</goal>
                        </goals>
                        <configuration>
                            <transformers>
                                <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
                                    <resource>reference.conf</resource>
                                </transformer>
                            </transformers>
                            <filters>
                                <filter>
                                    <artifact>*:*</artifact>
                                    <excludes>
                                        <exclude>META-INF/*.SF</exclude>
                                        <exclude>META-INF/*.DSA</exclude>
                                        <exclude>META-INF/*.RSA</exclude>
                                    </excludes>
                                </filter>
                            </filters>
                            <shadedArtifactAttached>true</shadedArtifactAttached>
                            <shadedClassifierName>jar-with-dependencies</shadedClassifierName>
                        </configuration>
                    </execution>
                </executions>
            </plugin>
        </plugins>
    </build>
    <dependencies>
        <!-- https://mvnrepository.com/artifact/org.quartz-scheduler/quartz -->
        <dependency>
            <groupId>org.quartz-scheduler</groupId>
            <artifactId>quartz</artifactId>
            <version>2.2.1</version>
        </dependency>
        <dependency>
            <groupId>com.floragunn</groupId>
            <artifactId>search-guard-5</artifactId>
            <version>5.2.2-15</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/com.floragunn/search-guard-ssl -->
        <dependency>
            <groupId>com.floragunn</groupId>
            <artifactId>search-guard-ssl</artifactId>
            <version>5.2.2-22</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/log4j/log4j -->
        <dependency>
            <groupId>org.elasticsearch.client</groupId>
            <artifactId>transport</artifactId>
            <version>5.2.2</version>
        </dependency>
        <dependency>
            <groupId>org.apache.logging.log4j</groupId>
            <artifactId>log4j-api</artifactId>
            <version>2.7</version>
        </dependency>
        <dependency>
            <groupId>org.apache.logging.log4j</groupId>
            <artifactId>log4j-core</artifactId>
            <version>2.7</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/org.slf4j/slf4j-log4j12 -->
        <dependency>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-log4j12</artifactId>
            <version>1.6.6</version>
        </dependency>
    </dependencies>
</project>

2、调用方法：

使用mvn进行打包部署后，进入target目录，输入nohup java -cp /root/remoa/indexTest/target/indexTest-1.0-SNAPSHOT-jar-with-dependencies.jar com.remoa.quartzCli.QuartzCli &将其设置为后台进程即可。