elk笔记22.1--通过api快速创建索引

1 简介

elk 是一个被广泛使用的日志检索系统，每次有新日志接入的时候，都需要一系列操作创建索引和kibana 配置。如果在企业中每次手动创建就显得很低效了，因此本文将相关操作封装成一个简单的python api，每次只需要传入指定的参数就可以快速完成索引创建和kibana 配置了。

2 功能实现

本文前置条件需要安装一套elk实例，具体安装方法可以参考笔者博文：
elk笔记1–搭建elk集群
elk笔记2–使用docker启一套elk实例

本文涉及的索引创建流程包括如下内容：

1. 新建一个索引生命周期策略 ilp_name(一般集群可以预先内置几个常用的策略)
2. 新建索引模板
3. 将索引模板和ilp_name相关联
4. 新建rollover类型的索引
5. 创建kibana index pattern

2.1 源码

#!/usr/bin/python
# -*- coding:utf-8 -*-

import requests
from base64 import b64encode


def get_base64_str(str_text):
    str_user_pwd = b64encode(str_text.encode('ascii'))
    return str_user_pwd.decode('ascii')


class ElasticHelper:
    def __init__(self, es_url, es_user, es_pwd, es_index_name, es_ilp_name):
        self.url = es_url
        self.user = es_user
        self.pwd = es_pwd
        self.index_name = es_index_name
        self.ilp_name = es_ilp_name

    def get_index_template(self):
        print(f"get_index_template: {self.index_name}")
        url_full = self.url + f"/_template/{self.index_name}"
        re = requests.get(url_full, auth=(self.user, self.pwd))
        if re.status_code == 200:
            return re.json()
        else:
            print("request error, not 200")
            return None

    def put_index_template(self):
        body = {'order': 0,
                'index_patterns': [f"{self.index_name}-*"],
                'settings': {
                    'index': {
                        'number_of_shards': '2',
                        'number_of_replicas': '0'
                    }
                },
                'mappings': {
                    "properties": {
                        "@timestamp": {
                            "type": "date"
                        }
                    }
                },
                'aliases': {}
        }
        url_full = self.url + f"/_template/{self.index_name}"
        re = requests.put(url_full, json=body, auth=(self.user, self.pwd))
        if re.status_code == 200:
            print(re.json())
        else:
            print("request error, not 200")

    def update_index_template_with_ilp(self):
        ret = self.get_index_template()
        if ret is None:
            print(f"index template {self.index_name}-* not exist, please create one!")
        body = ret[self.index_name]
        body['settings']['index']['lifecycle'] = {'name': f'{self.ilp_name}',
                                                  'rollover_alias': f'{self.index_name}_write'}
        re = requests.post(f"{self.url}/_template/{self.index_name}", json=body, auth=('elastic', 'elastic'))
        if re.status_code == 200:
            print(f"put_index_template(index={self.index_name},ilp={self.ilp_name}) succeed:\ninfo {re.status_code}, "
                  f"{re.text}")
        else:
            print(f"put_index_template(index={self.index_name},ilp={self.ilp_name}) failed:\ninfo {re.status_code}, "
                  f"{re.text}")

    def create_rollover_index(self):
        body = {
            "aliases": {
                f"{self.index_name}_write": {}
            }
        }
        re = requests.put(url=self.url+f"/%3C{self.index_name}-%7Bnow%2Fd%7D-000001%3E", json=body, auth=(self.user,
                                                                                                          self.pwd))
        if re.status_code == 200:
            print(re.json())
        else:
            print(f"create rollover index {self.index_name} failed!\n{re.status_code},\n{re.text}")

    def judge_alias_exist(self):
        re = requests.get(f"{self.url}/_cat/aliases", auth=('elastic', 'elastic'))
        if re.status_code == 200:
            ret_str = re.text
            if ret_str.find(f"{self.index_name}_write") != -1:
                return True
            else:
                False
        else:
            print(f"judge_alias_exist {self.index_name}_write failed!\n{re.status_code},\n{re.text}")
            return False


class KibanaHelper:
    def __init__(self, kibana_url, kibana_user, kibana_pwd, kibana_index_pattern_name):
        self.url = kibana_url
        self.user = kibana_user
        self.pwd = kibana_pwd
        self.index_pattern_name = kibana_index_pattern_name

    def get_index_pattern_id(self):
        url = f'{self.url}/api/saved_objects/_find?fields=title&fields=type&per_page=10000&type=index-pattern'
        headers = {'Authorization': 'Basic ' + get_base64_str(self.user + ':' + self.pwd)}
        ret = requests.get(url=url, headers=headers)
        for item in (ret.json())['saved_objects']:
            if item['attributes']['title'] == self.index_pattern_name:
                return item['id']
        return None

    def get_kibana_index_pattern(self):
        print(f"get_kibana_index_pattern {index_pattern_name}-*")
        headers = {'Authorization': 'Basic ' + get_base64_str(self.user + ':' + self.pwd), 'kbn-xsrf': 'reporting'}
        id = self.get_index_pattern_id()
        url_full = self.url + f"/api/saved_objects/_bulk_get"
        body = [{"id": id, "type": "index-pattern"}]
        re = requests.post(url=url_full, json=body, headers=headers)
        if re.status_code == 200:
            print(re.text)
        else:
            print(f"error:\n {re.text}")

    def create_kibana_index_pattern(self):
        print(f"index_pattern={self.index_pattern_name}")
        ret = self.get_index_pattern_id()
        if ret is not None:
            print(f"{self.index_pattern_name} has been existed, please do not repeat")
            exit(0)
        headers = {'Authorization': 'Basic ' + get_base64_str(self.user + ':' + self.pwd), 'kbn-xsrf': 'reporting'}
        url_full = self.url + "/api/saved_objects/index-pattern"
        body = {
            "attributes": {
                "title": self.index_pattern_name,
                "timeFieldName": "@timestamp"
            }
        }
        re = requests.post(url_full, json=body, headers=headers)
        if re.status_code == 200:
            print(re.text)
        else:
            print(f"error:\n {re.text}")

    def delete_index_pattern_id(self, index_pattern_id):
        print(f"delete index_pattern_id={index_pattern_id}")
        headers = {'Authorization': 'Basic ' + get_base64_str(self.user + ':' + self.pwd), 'kbn-xsrf': 'reporting'}
        url_full = self.url + f"/api/saved_objects/index-pattern/{index_pattern_id}?force=false"
        re = requests.delete(url_full, headers=headers)
        if re.status_code == 200:
            print(f"delete_index_pattern_id({index_pattern_id}) succeed:\ninfo {re.status_code}, {re.text}")
        else:
            print(f"delete_index_pattern_id({index_pattern_id}) error:\ninfo {re.status_code}, {re.text}")

    def get_ilm(self):
        url_full = self.url+"/api/index_lifecycle_management/policies?withIndices=true"
        headers = {'Authorization': 'Basic ' + get_base64_str(self.user + ':' + self.pwd), 'kbn-xsrf': 'reporting'}
        re = requests.get(url_full, headers=headers)
        if re.status_code == 200:
            print(f"get_ilm succeed")
            return re.json()
        else:
            print(f"get_ilm failed\ninfo {re.status_code},{re.text}")
            return None

    def judge_ilp_exist(self, ilp_name):
        ilp_list = self.get_ilm()
        if ilp_list is None:
            return False
        else:
            for item in ilp_list:
                if ilp_name == item["name"]:
                    return True
            return False


if __name__ == "__main__":
    es_url = "http://127.0.0.1:9200"
    es_user = "elastic"
    es_pwd = "elastic"
    kibana_url = "http://127.0.0.1:5601"
    kibana_user = "elastic"
    kibana_pwd = "elastic"
    index_name = "test-elk"
    index_pattern_name = index_name + "-*"
    ilp_name = "ilp-default"
    es = ElasticHelper(es_url, es_user, es_pwd, index_name, ilp_name)
    kibana = KibanaHelper(kibana_url, kibana_user, kibana_pwd, index_pattern_name)
    es.put_index_template()
    if not kibana.judge_ilp_exist(ilp_name):
        print(f"ilp_name={ilp_name} not exist, please change another one")
        exit(0)
    es.update_index_template_with_ilp()
    if es.judge_alias_exist():
        print(f"alias_name={index_name}_write exists, please change another one")
        exit(0)
    es.create_rollover_index()
    kibana.create_kibana_index_pattern()

2.2 测试

在kibana->Stack Management->Index Lifecycle Policies->新建ilp-default

执行结果：

向kibana写一条数据：

POST test-elk_write/_doc
{
  "@timestamp": "2021-08-29T19:00:00+08:00",
  "name": "test001"
}
此处需要设置+08:00, 否则查看的时间会向后推8小时

kibana 查看数据：

3 注意事项

1. 7.10.x 及以下的版本对kibana index pattern 的api 支持不完善，因此需要自己通过浏览器捕获api和传递的参数。
   

2. 访问kibana 的时候header中需要添加 ‘kbn-xsrf’: ‘reporting’ 参数，否则会报错
   具体信息见 Request must contain a kbn-xsrf header

4 说明

软件环境：
ubuntu 20.04 desktop
elk 7.10.0(docker)
python 3.8
参考文档：
1 elasticsearch/reference/current/index.html
2 kibana/current/index.html
3 elk笔记1–搭建elk集群
4 elk笔记2–使用docker启一套elk实例
5 Request must contain a kbn-xsrf header