1 简介
elk 是一个被广泛使用的日志检索系统,每次有新日志接入的时候,都需要一系列操作创建索引和kibana 配置。如果在企业中每次手动创建就显得很低效了,因此本文将相关操作封装成一个简单的python api,每次只需要传入指定的参数就可以快速完成索引创建和kibana 配置了。
2 功能实现
本文前置条件需要安装一套elk实例,具体安装方法可以参考笔者博文:
elk笔记1–搭建elk集群
elk笔记2–使用docker启一套elk实例
本文涉及的索引创建流程包括如下内容:
- 新建一个索引生命周期策略 ilp_name(一般集群可以预先内置几个常用的策略)
- 新建索引模板
- 将索引模板和ilp_name相关联
- 新建rollover类型的索引
- 创建kibana index pattern
2.1 源码
#!/usr/bin/python
# -*- coding:utf-8 -*-
import requests
from base64 import b64encode
def get_base64_str(str_text):
str_user_pwd = b64encode(str_text.encode('ascii'))
return str_user_pwd.decode('ascii')
class ElasticHelper:
def __init__(self, es_url, es_user, es_pwd, es_index_name, es_ilp_name):
self.url = es_url
self.user = es_user
self.pwd = es_pwd
self.index_name = es_index_name
self.ilp_name = es_ilp_name
def get_index_template(self):
print(f"get_index_template: {self.index_name}")
url_full = self.url + f"/_template/{self.index_name}"
re = requests.get(url_full, auth=(self.user, self.pwd))
if re.status_code == 200:
return re.json()
else:
print("request error, not 200")
return None
def put_index_template(self):
body = {'order': 0,
'index_patterns': [f"{self.index_name}-*"],
'settings': {
'index': {
'number_of_shards': '2',
'number_of_replicas': '0'
}
},
'mappings': {
"properties": {
"@timestamp": {
"type": "date"
}
}
},
'aliases': {}
}
url_full = self.url + f"/_template/{self.index_name}"
re = requests.put(url_full, json=body, auth=(self.user, self.pwd))
if re.status_code == 200:
print(re.json())
else:
print("request error, not 200")
def update_index_template_with_ilp(self):
ret = self.get_index_template()
if ret is None:
print(f"index template {self.index_name}-* not exist, please create one!")
body = ret[self.index_name]
body['settings']['index']['lifecycle'] = {'name': f'{self.ilp_name}',
'rollover_alias': f'{self.index_name}_write'}
re = requests.post(f"{self.url}/_template/{self.index_name}", json=body, auth=('elastic', 'elastic'))
if re.status_code == 200:
print(f"put_index_template(index={self.index_name},ilp={self.ilp_name}) succeed:\ninfo {re.status_code}, "
f"{re.text}")
else:
print(f"put_index_template(index={self.index_name},ilp={self.ilp_name}) failed:\ninfo {re.status_code}, "
f"{re.text}")
def create_rollover_index(self):
body = {
"aliases": {
f"{self.index_name}_write": {}
}
}
re = requests.put(url=self.url+f"/%3C{self.index_name}-%7Bnow%2Fd%7D-000001%3E", json=body, auth=(self.user,
self.pwd))
if re.status_code == 200:
print(re.json())
else:
print(f"create rollover index {self.index_name} failed!\n{re.status_code},\n{re.text}")
def judge_alias_exist(self):
re = requests.get(f"{self.url}/_cat/aliases", auth=('elastic', 'elastic'))
if re.status_code == 200:
ret_str = re.text
if ret_str.find(f"{self.index_name}_write") != -1:
return True
else:
False
else:
print(f"judge_alias_exist {self.index_name}_write failed!\n{re.status_code},\n{re.text}")
return False
class KibanaHelper:
def __init__(self, kibana_url, kibana_user, kibana_pwd, kibana_index_pattern_name):
self.url = kibana_url
self.user = kibana_user
self.pwd = kibana_pwd
self.index_pattern_name = kibana_index_pattern_name
def get_index_pattern_id(self):
url = f'{self.url}/api/saved_objects/_find?fields=title&fields=type&per_page=10000&type=index-pattern'
headers = {'Authorization': 'Basic ' + get_base64_str(self.user + ':' + self.pwd)}
ret = requests.get(url=url, headers=headers)
for item in (ret.json())['saved_objects']:
if item['attributes']['title'] == self.index_pattern_name:
return item['id']
return None
def get_kibana_index_pattern(self):
print(f"get_kibana_index_pattern {index_pattern_name}-*")
headers = {'Authorization': 'Basic ' + get_base64_str(self.user + ':' + self.pwd), 'kbn-xsrf': 'reporting'}
id = self.get_index_pattern_id()
url_full = self.url + f"/api/saved_objects/_bulk_get"
body = [{"id": id, "type": "index-pattern"}]
re = requests.post(url=url_full, json=body, headers=headers)
if re.status_code == 200:
print(re.text)
else:
print(f"error:\n {re.text}")
def create_kibana_index_pattern(self):
print(f"index_pattern={self.index_pattern_name}")
ret = self.get_index_pattern_id()
if ret is not None:
print(f"{self.index_pattern_name} has been existed, please do not repeat")
exit(0)
headers = {'Authorization': 'Basic ' + get_base64_str(self.user + ':' + self.pwd), 'kbn-xsrf': 'reporting'}
url_full = self.url + "/api/saved_objects/index-pattern"
body = {
"attributes": {
"title": self.index_pattern_name,
"timeFieldName": "@timestamp"
}
}
re = requests.post(url_full, json=body, headers=headers)
if re.status_code == 200:
print(re.text)
else:
print(f"error:\n {re.text}")
def delete_index_pattern_id(self, index_pattern_id):
print(f"delete index_pattern_id={index_pattern_id}")
headers = {'Authorization': 'Basic ' + get_base64_str(self.user + ':' + self.pwd), 'kbn-xsrf': 'reporting'}
url_full = self.url + f"/api/saved_objects/index-pattern/{index_pattern_id}?force=false"
re = requests.delete(url_full, headers=headers)
if re.status_code == 200:
print(f"delete_index_pattern_id({index_pattern_id}) succeed:\ninfo {re.status_code}, {re.text}")
else:
print(f"delete_index_pattern_id({index_pattern_id}) error:\ninfo {re.status_code}, {re.text}")
def get_ilm(self):
url_full = self.url+"/api/index_lifecycle_management/policies?withIndices=true"
headers = {'Authorization': 'Basic ' + get_base64_str(self.user + ':' + self.pwd), 'kbn-xsrf': 'reporting'}
re = requests.get(url_full, headers=headers)
if re.status_code == 200:
print(f"get_ilm succeed")
return re.json()
else:
print(f"get_ilm failed\ninfo {re.status_code},{re.text}")
return None
def judge_ilp_exist(self, ilp_name):
ilp_list = self.get_ilm()
if ilp_list is None:
return False
else:
for item in ilp_list:
if ilp_name == item["name"]:
return True
return False
if __name__ == "__main__":
es_url = "http://127.0.0.1:9200"
es_user = "elastic"
es_pwd = "elastic"
kibana_url = "http://127.0.0.1:5601"
kibana_user = "elastic"
kibana_pwd = "elastic"
index_name = "test-elk"
index_pattern_name = index_name + "-*"
ilp_name = "ilp-default"
es = ElasticHelper(es_url, es_user, es_pwd, index_name, ilp_name)
kibana = KibanaHelper(kibana_url, kibana_user, kibana_pwd, index_pattern_name)
es.put_index_template()
if not kibana.judge_ilp_exist(ilp_name):
print(f"ilp_name={ilp_name} not exist, please change another one")
exit(0)
es.update_index_template_with_ilp()
if es.judge_alias_exist():
print(f"alias_name={index_name}_write exists, please change another one")
exit(0)
es.create_rollover_index()
kibana.create_kibana_index_pattern()
2.2 测试
在kibana->Stack Management->Index Lifecycle Policies->新建ilp-default
执行结果:
向kibana写一条数据:
POST test-elk_write/_doc
{
"@timestamp": "2021-08-29T19:00:00+08:00",
"name": "test001"
}
此处需要设置+08:00, 否则查看的时间会向后推8小时
kibana 查看数据:
3 注意事项
-
7.10.x 及以下的版本对kibana index pattern 的api 支持不完善,因此需要自己通过浏览器捕获api和传递的参数。
-
访问kibana 的时候header中需要添加 ‘kbn-xsrf’: ‘reporting’ 参数,否则会报错
具体信息见 Request must contain a kbn-xsrf header
4 说明
软件环境:
ubuntu 20.04 desktop
elk 7.10.0(docker)
python 3.8
参考文档:
1 elasticsearch/reference/current/index.html
2 kibana/current/index.html
3 elk笔记1–搭建elk集群
4 elk笔记2–使用docker启一套elk实例
5 Request must contain a kbn-xsrf header