zabbix监控elasticsearch集群

今天同事负责的es集群发生了脑裂，具体原因还有待查看日志。
顺便分享一套zabbix监控es集群的脚本。

生产改进与建议：

• 所有监控统一status值，比如0是ok的，1是警告，2是error
• 因为es集群会自己维护整个集群的元数据，因此数据收集不是按节点来的而是整个集群
• 现在的配置是从salt的pillar中获取端口（或者说集群名）然后渲染下面的脚本，然后再自动发现集群下面的节点。建议集群也使用脚本自动检查，特别是一个主机上有多个属于不同集群的es节点时。
• es内部gc的平均时间没有计算出来，后期可以加
• 最后还是脚本中要收集的数据还是用字符串的+，也在此收集更高效的处理方法

架构方面：

• 初步估计es集群脑裂不外乎两个原因
  • 网络原因，适当调整es集群内节点发现超时时间
  • master节点和data节点混在一起，当data节点由于业务原因hang住时，可能会导致集群将此节点剔除，引起es集群重新选举。
• 解决：
  • master节点和data节点分离，使用3个master节点，其余做data节点，master节点的jvm配置可以配置小一点。es集群的监控放在master节点上。

#!/usr/bin/env python
#coding:utf-8

from __future__ import division   #必须在第一行
import json,requests,sys,os
import pickle



#在os.popen中执行的时候\b要加一层\
#如果没有指定ip，就使用默认的，用系统命令抓取本地ip地址：
cmd = "ip addr|grep '\\binet\\b'|grep -E 'bond0|eth0'|awk '{print $2}'|awk -F'/' '{print $1}'"
local_ip = os.popen(cmd).read().strip()

#设置一个默认端口9200,这个后期使用自动发现比较好
local_port = 9200


def discovery(local_ip,local_port):
    r = {}
    r['data'] = []

    res = requests.get("http://{0}:{1}/_cat/nodes?v&h=name".format(local_ip,local_port))
    if res.status_code == 200:
        ret = res.text.splitlines()
        for i in range(1,len(ret)):
            r['data'].append({'{#NODE}':ret[i]})
    return json.dumps(r)


def send(local_ip,local_port):
    r_str = ""
    zbx_sender_cmd = "{0} -c {1} -i {2}"
    zbx_conf = "/usr/local/services/zabbix-3.0.0/etc/zabbix_agentd.conf"
    zbx_sender_file = "/tmp/.zbx_elastic_sender.txt"
    zbx_sender = "/usr/local/services/zabbix-3.0.0/bin/zabbix_sender"
    last_file = "/usr/local/services/zabbix-3.0.0/etc/.zbx_elstic_last_data.txt"

    #需要pickle保存到文件中的字典
    this_data = {}

    #导入历史数据
    if os.path.exists(last_file):
        with open(last_file,"r") as f1:
            last_data = pickle.load(f1)
    else:
        os.popen("touch /usr/local/services/zabbix-3.0.0/etc/.zbx_elstic_last_data.txt")
        last_data = {}
    #print(last_data)

    # 获取node信息的url
    url_node = "http://{0}:{1}/_nodes/stats?pretty".format(local_ip, local_port)
    res_node = requests.get(url_node)
    if res_node.status_code == 200:
        ret_node = res_node.json()
        for node,node_value in ret_node["nodes"].items():
            #print(ret_node["nodes"][node]["name"])  #打印出节点名字
            node_name = node_value["name"]
            this_data[node_name] = {}

            #需要获取docs,segments,get,search,merges,flush,warmer等信息
            #docs,segments
            r_str += "- elastic.indices.docs.count.[{0}] {1}\n".format(node_name,node_value["indices"]["docs"]["count"])
            r_str += "- elastic.indices.docs.deleted.[{0}] {1}\n".format(node_name, node_value["indices"]["docs"]["deleted"])
            r_str += "- elastic.indices.segments.count.[{0}] {1}\n".format(node_name, node_value["indices"]["segments"]["count"])
            r_str += "- elastic.indices.segments.memory.[{0}] {1}\n".format(node_name,node_value["indices"]["segments"]["memory_in_bytes"])

            #indexing,get,search
            indexing_num = node_value["indices"]["indexing"]["index_total"]
            indexing_time = node_value["indices"]["indexing"]["index_time_in_millis"]/1000
            r_str += "- elastic.indices.indexing.total.[{0}] {1}\n".format(node_name, indexing_num)
            this_data[node_name].update({"indexing_num": indexing_num})
            r_str += "- elastic.indices.indexing.time.[{0}] {1}\n".format(node_name, indexing_time)
            this_data[node_name].update({"indexing_time": indexing_time})
            if node_name not in last_data.keys() or (indexing_num - last_data[node_name]["indexing_num"]) == 0:
                r_str += "- elastic.indices.indexing.per_time.[{0}] {1}\n".format(node_name,0)
            else:
                r_str += "- elastic.indices.indexing.per_time.[{0}] {1}\n".format(node_name,
                        round((indexing_time - last_data[node_name]["indexing_time"])/(indexing_num - last_data[node_name]["indexing_num"]),3)
                                                                          )

            get_num = node_value["indices"]["get"]["total"]
            get_time = node_value["indices"]["get"]["time_in_millis"]/1000
            r_str += "- elastic.indices.get.total.[{0}] {1}\n".format(node_name, get_num)
            this_data[node_name].update({"get_num": get_num})
            r_str += "- elastic.indices.get.time.[{0}] {1}\n".format(node_name, get_time)
            this_data[node_name].update({"get_time": get_time})
            if node_name not in last_data.keys() or (get_num - last_data[node_name]["get_num"]) == 0:
                r_str += "- elastic.indices.get.per_time.[{0}] {1}\n".format(node_name,0)
            else:
                r_str += "- elastic.indices.get.per_time.[{0}] {1}\n".format(node_name,
                        round((get_time - last_data[node_name]["get_time"])/(get_num - last_data[node_name]["get_num"]),3)
                                                                     )

            query_num = node_value["indices"]["search"]["query_total"]
            query_time = node_value["indices"]["search"]["query_time_in_millis"]/1000
            r_str += "- elastic.indices.search.query_total.[{0}] {1}\n".format(node_name, query_num)
            this_data[node_name].update({"query_num": query_num})
            r_str += "- elastic.indices.search.query_time.[{0}] {1}\n".format(node_name, query_time)
            this_data[node_name].update({"query_time": query_time})
            if node_name not in last_data.keys() or (query_num - last_data[node_name]["query_num"]) == 0:
                r_str += "- elastic.indices.query.per_time.[{0}] {1}\n".format(node_name,0)
            else:
                r_str += "- elastic.indices.query.per_time.[{0}] {1}\n".format(node_name,
                        round((query_time - last_data[node_name]["query_time"])/(query_num - last_data[node_name]["query_num"]),3)
                                                                         )

            fetch_num = node_value["indices"]["search"]["fetch_total"]
            fetch_time = node_value["indices"]["search"]["fetch_time_in_millis"]/1000
            r_str += "- elastic.indices.search.fetch_total.[{0}] {1}\n".format(node_name, fetch_num)
            this_data[node_name].update({"fetch_num": fetch_num})
            r_str += "- elastic.indices.search.fetch_time.[{0}] {1}\n".format(node_name,fetch_time)
            this_data[node_name].update({"fetch_time": fetch_time})
            if node_name not in last_data.keys() or (fetch_num - last_data[node_name]["fetch_num"]) == 0:
                r_str += "- elastic.indices.fetch.per_time.[{0}] {1}\n".format(node_name,0)
            else:
                r_str += "- elastic.indices.fetch.per_time.[{0}] {1}\n".format(node_name,
                        round((fetch_time - last_data[node_name]["fetch_time"])/(fetch_num - last_data[node_name]["fetch_num"]),3)
                                                                         )

            #merges,refresh,flush,warmer
            for oper in ["merges","refresh","flush","warmer"]:
                #这里有点容易混淆
                p_data = {}
                str_num = "{0}_num".format(oper)
                str_time = "{0}_time".format(oper)
                p_data[str_num] = node_value["indices"][oper]["total"]
                p_data[str_time] = node_value["indices"][oper]["total_time_in_millis"]/1000

                r_str += "- elastic.indices.{0}.total.[{1}] {2}\n".format(oper,node_name, p_data[str_num])
                this_data[node_name].update({"{0}_num".format(oper): p_data[str_num]})
                r_str += "- elastic.indices.{0}.time.[{1}] {2}\n".format(oper,node_name, p_data[str_time])
                this_data[node_name].update({"{0}_time".format(oper): p_data[str_time]})
                if node_name not in last_data.keys() or (p_data[str_num] - last_data[node_name][str_num]) == 0:
                    r_str += "- elastic.indices.{0}.per_time.[{1}] {2}\n".format(oper,node_name, 0)
                else:
                    r_str += "- elastic.indices.{0}.per_time.[{1}] {2}\n".format(oper,node_name,
                                round((p_data[str_time] - last_data[node_name][str_time]) / (p_data[str_num] - last_data[node_name][str_num]),3)
                                                                               )

            #jvm基本
            r_str += "- elastic.jvm.heap_max_in_bytes.[{0}] {1}\n".format(node_name,node_value["jvm"]["mem"]["heap_max_in_bytes"])
            r_str += "- elastic.jvm.heap_used_in_bytes.[{0}] {1}\n".format(node_name,node_value["jvm"]["mem"]["heap_used_in_bytes"])
            r_str += "- elastic.jvm.threads.[{0}] {1}\n".format(node_name,node_value["jvm"]["threads"]["count"])
            r_str += "- elastic.jvm.buffer_pools.used_in_bytes.[{0}] {1}\n".format(node_name,node_value["jvm"]["buffer_pools"]["direct"]["used_in_bytes"])
            r_str += "- elastic.jvm.buffer_pools.total_capacity_in_bytes.[{0}] {1}\n".format(node_name,node_value["jvm"]["buffer_pools"]["direct"]["total_capacity_in_bytes"])

            #jvm垃圾回收的两个总量，要用减法。后面改进要计算出平均垃圾回收的时间
            r_str += "- elastic.jvm.gc.young.num.[{0}] {1}\n".format(node_name,node_value["jvm"]["gc"]["collectors"]["young"]["collection_count"])
            r_str += "- elastic.jvm.gc.young.time.[{0}] {1}\n".format(node_name,node_value["jvm"]["gc"]["collectors"]["young"]["collection_time_in_millis"]/1000)
            r_str += "- elastic.jvm.gc.old.num.[{0}] {1}\n".format(node_name,node_value["jvm"]["gc"]["collectors"]["old"]["collection_count"])
            r_str += "- elastic.jvm.gc.old.time.[{0}] {1}\n".format(node_name,node_value["jvm"]["gc"]["collectors"]["old"]["collection_time_in_millis"]/1000)

            for m,n in node_value["jvm"]["mem"]["pools"].items():
                r_str += "- elastic.jvm.{0}.used_in_bytes.[{1}] {2}\n".format(m,node_name,n["used_in_bytes"])
                r_str += "- elastic.jvm.{0}.max_in_bytes.[{1}] {2}\n".format(m,node_name,n["max_in_bytes"])

            #thread_pool，这个东西正常情况下没什么，有些时候还是能发现很多问题的
            for k,v in node_value["thread_pool"].items():
                r_str += "- elastic.thread_pool.{0}.threads.[{1}] {2}\n".format(k,node_name,v[u"threads"])
                r_str += "- elastic.thread_pool.{0}.threads.queue.[{1}] {2}\n".format(k,node_name,v[u"queue"])

            #http和script，其中script是总的数据，做减法
            r_str += "- elastic.http.current_open.[{0}] {1}\n".format(node_name,node_value["http"]["current_open"])
            r_str += "- elastic.script.compilations.[{0}] {1}\n".format(node_name,node_value["script"]["compilations"])

            #处理每种操作的平均时间time差值/total差值
            with open(last_file,"w") as f2:
                pickle.dump(this_data,f2)



    else:
        sys.stderr.write("Fetch node info error!")


    # 获取集群信息的url
    url_cluster = "http://{0}:{1}/_cluster/health".format(local_ip, local_port)
    res_cluster = requests.get(url_cluster)
    if res_cluster.status_code == 200:
        ret_cluster = res_cluster.json()

        #print(ret_cluster)
        #绿是0，黄是1，红是2
        if ret_cluster[u'status'] == u"green":
            status = 0
        elif ret_cluster[u'status'] == u"yellow":
            status = 1
        else:
            status = 2

        r_str +=  "- elastic.cluster.status {0}\n".format(status)
        r_str += "- elastic.cluster.non {0}\n".format(ret_cluster[u"number_of_nodes"])
        r_str += "- elastic.cluster.us {0}\n".format(ret_cluster[u"unassigned_shards"])
        r_str += "- elastic.cluster.nopt {0}\n".format(ret_cluster[u"number_of_pending_tasks"])
        r_str += "- elastic.cluster.noiff {0}\n".format(ret_cluster[u"number_of_in_flight_fetch"])
        r_str += "- elastic.cluster.aps {0}\n".format(ret_cluster[u"active_primary_shards"])
        r_str += "- elastic.cluster.tmwiqm {0}\n".format(ret_cluster[u"task_max_waiting_in_queue_millis"])
        r_str += "- elastic.cluster.rs {0}\n".format(ret_cluster[u"relocating_shards"])
        r_str += "- elastic.cluster.aspan {0}\n".format(ret_cluster[u"active_shards_percent_as_number"])
        r_str += "- elastic.cluster.as {0}\n".format(ret_cluster[u"active_shards"])
        r_str += "- elastic.cluster.is {0}\n".format(ret_cluster[u"initializing_shards"])
        r_str += "- elastic.cluster.dus {0}\n".format(ret_cluster[u"delayed_unassigned_shards"])
        r_str += "- elastic.cluster.nodn {0}\n".format(ret_cluster[u"number_of_data_nodes"])
    else:
        sys.stderr.write("Fetch node info error!")

    with open(zbx_sender_file,"w") as f:
        f.write(r_str)


    send_ret = os.popen(zbx_sender_cmd.format(zbx_sender, zbx_conf, zbx_sender_file))
    #print(zbx_sender_cmd.format(zbx_sender, zbx_conf, zbx_sender_file))
    if "failed: 0" in send_ret.read():  #这一步，用一个普通的item来触发，并返回执行结果，1是正常的，0是发送异常
        print(1)
    else:
        print(0)



if __name__ == "__main__":
    if len(sys.argv) == 2 and sys.argv[1]=="discovery":
        ret = discovery(local_ip,local_port)
        print(ret)
    elif len(sys.argv) == 1:
        send(local_ip,local_port)
    else:
        sys.stderr.write("Args is wrong!")