ES快照备份以及索引定时清除（python）

说明：提前创建好本地快照仓库（数据量大的情况下也可以连接到其它比如：hdfs），对前一天的数据做快照，并删除历史数据和过期的快照数据。

创建快照仓库

PUT /_snapshot/es_backup
{
“type”: “fs”,
“settings”: {
“compress” : “true”,
“max_snapshot_bytes_per_sec” : “100mb”,
“max_restore_bytes_per_sec” : “100mb”,
“location”: “/data/es_backup”
}
}

更新配置文件elasticsearch.yml，添加以下配置：

path.repo: ["/data/es_backup"]

创建快照以索引删除策略
python es_snapshots.py -h ##查看帮助信息
python es_snapshots.py --delete_index=<参数> ##删除n月前的一个快照

# es_snapshot.py

# -*- coding: utf-8 -*-

import traceback
from elasticsearch import Elasticsearch
import re
import datetime ,time
from dateutil.relativedelta import relativedelta
import argparse

class EsSnapshot():
    def __init__(self, Es):
        self.Es = Es

    @staticmethod
    def date_range(date_s):
        import datetime, time
        now_date = datetime.datetime.fromtimestamp(time.time()).strftime('%Y%m')
        from datetime import datetime
        del_range = str(datetime.date(datetime.strptime(now_date, '%Y%m'))
                        - relativedelta(months=date_s)).replace("-", '')[:-2]
        return del_range

    def indices(self,date_s,index_pre):
        self.Es.indices.get_alias()
        ###输出日期范围内索引
        indices_list = []
        del_data_list = []
        indices = self.Es.indices.get_alias().keys()
        for index in indices:
            if re.search('^{}'.format(index_pre), index):
                indices_list.append(index)
        for data in indices_list:
            source_m = re.search("(\d{6})", data)
            all_index = source_m.group()
            del_range = EsSnapshot.date_range(date_s)
            if int(all_index) == int(del_range):
                del_data_list.append(data)
        return del_data_list

    def create_snapshot(self, *args):
        ###获取前一天日期的索引做快照备份
        create_sna_index =self.indices(args[0], args[1])
        print ("创建快照：snapshot_%s" % (create_sna_index))
        self.Es.snapshot.create(repository="es_backup",
                                snapshot="snapshot_{}".format(create_sna_index),
                                body=create_sna_index)

    def Del_index(self, *args):
        ###匹配索引删除
        if args[1] is not None:
            del_status = input(
                "是否删除'{}'索引，yes/no：".format(args[1]))
            if del_status == "yes":
                print("删除索引{}".format(args[1]))
                self.Es.indices.delete(args[1])
        elif args[0] is not None:
            del_index = self.indices(int(args[0]),
                                     args[2])
            del_status = input(
                "是否删除'{}'索引，yes/no：".format(''.join(del_index)))
            if del_status == "yes" and str(del_index).strip() != '':
                print("删除索引{}".format(''.join(del_index)))
                self.Es.indices.delete(del_index)

    def Del_snapshot(self, *args):
        ###匹配快照删除
        del_index_sna =self.indices(args[0],args[1])
        index_body = "snapshot_{}".format(del_index_sna)
        self.Es.snapshot.delete(repository="es_backup", snapshot=index_body)
        print("删除快照：%s" %(index_body))

if __name__ == '__main__':
    '''集群连接'''
    Es = Elasticsearch("http://ip:9200",
                       http_auth=('elastic', 'xxxx'), port=9200)
    execute = EsSnapshot(Es)
    ###命令行传参
    parser = argparse.ArgumentParser()
    parser.add_argument('--index_pre', type=str, default=None, 
						help="匹配索引前缀，指定时间区间删除索引和创建快照必须使用")
    parser.add_argument('--delete_index_name', type=str, default=None, 
						help="指定索引名称删除,--delete_index_name=<参数>")
    parser.add_argument('--delete_index_month', type=int, default=None,
						help="删除n月前的第一个索引，--delete_index_month=<指定月份> --index_pre=<参数>")
    parser.add_argument('--create_snapashot', type=str, default=None, 
						help="创建快照，--create_snapashot=<指定月份> --index_pre=<参数>")
    parser.add_argument('--delete_snapashot', type=str, default=None, 
						help="删除N月前快照，--delete_snapashot=<指定月份> --index_pre=<参数>")
    args = parser.parse_args()
    try:
        if args.delete_index_month is not None or args.delete_index_name is not None:
            execute.Del_index(args.delete_index_month, args.delete_index_name, args.index_pre)
        elif args.create_snapashot is not None:
            execute.create_snapshot(args.create_snapashot,args.index_pre)
        elif args.delete_snapashot is not None:
            execute.Del_snapshot(args.delete_snapashot,args.index_pre)
    except Exception as e:
        print(traceback.print_exc())

快照恢复：

POST /_snapshot/es_backup/快照名称/_restore