8.ElasticSearch预警服务-Watcher详解-监控Marvel数据

8.ElasticSearch预警服务-Watcher详解-监控Marvel数据
如果ElasticSearch集群部署了Marvel相关的服务,那么就可以安装Watcher来监控异常情况的产生并发送预警。
例如可以在以下情况下配置Watch Action动作.
1.集群监控状态监控
2.高内存使用率
3.高CPU使用率
4.高文件目录空间
5.高FieldData 缓存使用
6.节点加入或者脱离集群
使用Watcher查询Marvel存储的集群信息数据,请注意Marvel数据是否存储正常。

 

1.监控集群状态配置:
 每分钟检测一次,如果集群状态持续Red状态60秒,则触发预警动作。

 

 PUT _watcher/watch/cluster_red_alert
{
  "trigger": {
    "schedule": {
      "interval": "1m"
    }
  },
  "input": {
    "search": {
      "request": {
        "indices": ".marvel-*",
        "types": "cluster_stats",
        "body": {
          "query": {
            "filtered": {
              "filter": {
                "bool": {
                  "must": [
                    {
                      "range": {
                        "@timestamp": {
                          "gte": "now-2m",
                          "lte": "now"
                        }
                      }
                    }
                  ],
                  "should": [
                    {
                      "term": {
                        "status.raw": "red"
                      }
                    },
                    {
                      "term": {
                        "status.raw": "green"
                      }
                    },
                    {
                      "term": {
                        "status.raw": "yellow"
                      }
                    }
                  ]
                }
              }
            }
          },
          "fields": ["@timestamp","status"],
          "sort": [
            {
              "@timestamp": {
                "order": "desc"
              }
            }
          ],
          "size": 1,
          "aggs": {
            "minutes": {
              "date_histogram": {
                "field": "@timestamp",
                "interval": "5s"
              },
              "aggs": {
                "status": {
                  "terms": {
                    "field": "status.raw",
                    "size": 3
                  }
                }
              }
            }
          }
        }
      }
    }
  },
  "throttle_period": "30m", 
  "condition": {
    "script": {
      "inline": "if (ctx.payload.hits.total < 1) return false; def rows = ctx.payload.hits.hits; if (rows[0].fields.status[0] != 'red') return false; if (ctx.payload.aggregations.minutes.buckets.size() < 12) return false; def last60Seconds = ctx.payload.aggregations.minutes.buckets[-12..-1]; return last60Seconds.every { it.status.buckets.every { s -> s.key == 'red' } }"
    }
  },
  "actions": {
    "send_email": { 
      "email": {
        "to": "<username>@<domainname>", 
        "subject": "Watcher Notification - Cluster has been RED for the last 60 seconds",
        "body": "Your cluster has been red for the last 60 seconds."
      }
    }
  }
}

 2.监控内存使用
每分钟检测一次,如果60秒内集群内存使用大于75%,则发送预警邮件

 

 

PUT _watcher/watch/mem_watch
{
  "trigger": {
    "schedule": {
      "interval": "1m"
    }
  },
  "input": {
    "search": {
      "request": {
        "indices": [
          ".marvel-*"
        ],
        "search_type": "count",
        "body": {
          "query": {
            "filtered": {
              "filter": {
                "range": {
                  "@timestamp": {
                    "gte": "now-2m",
                    "lte": "now"
                  }
                }
              }
            }
          },
          "aggs": {
            "minutes": {
              "date_histogram": {
                "field": "@timestamp",
                "interval": "minute"
              },
              "aggs": {
                "nodes": {
                  "terms": {
                    "field": "node.name.raw",
                    "size": 10,
                    "order": {
                      "memory": "desc"
                    }
                  },
                  "aggs": {
                    "memory": {
                      "avg": {
                        "field": "jvm.mem.heap_used_percent"
                      }
                    }
                  }
                }
              }
            }
          }
        }
      }
    }
  },
  "throttle_period": "30m", 
  "condition": {
    "script":  "if (ctx.payload.aggregations.minutes.buckets.size() == 0) return false; def latest = ctx.payload.aggregations.minutes.buckets[-1]; def node = latest.nodes.buckets[0]; return node && node.memory && node.memory.value >= 75;"
  },
  "actions": {
    "send_email": {
      "transform": {
        "script": "def latest = ctx.payload.aggregations.minutes.buckets[-1]; return latest.nodes.buckets.findAll { return it.memory && it.memory.value >= 75 };"
      },
      "email": { 
        "to": "<username>@<domainname>", 
        "subject": "Watcher Notification - HIGH MEMORY USAGE",
        "body": "Nodes with HIGH MEMORY Usage (above 75%):\n\n{{#ctx.payload._value}}\"{{key}}\" - Memory Usage is at {{memory.value}}%\n{{/ctx.payload._value}}"
      }
    }
  }
}

 3.CPU使用率监控
每分钟执行一次,如果60秒CPU使用率大于75%,则发送预警邮件。

 

PUT _watcher/watch/cpu_usage
{
  "trigger": {
    "schedule": {
      "interval": "1m"
    }
  },
  "input": {
    "search": {
      "request": {
        "indices": [
          ".marvel-*"
        ],
        "search_type": "count",
        "body": {
          "query": {
            "filtered": {
              "filter": {
                "range": {
                  "@timestamp": {
                    "gte": "now-2m",
                    "lte": "now"
                  }
                }
              }
            }
          },
          "aggs": {
            "minutes": {
              "date_histogram": {
                "field": "@timestamp",
                "interval": "minute"
              },
              "aggs": {
                "nodes": {
                  "terms": {
                    "field": "node.name.raw",
                    "size": 10,
                    "order": {
                      "cpu": "desc"
                    }
                  },
                  "aggs": {
                    "cpu": {
                      "avg": {
                        "field": "os.cpu.user"
                      }
                    }
                  }
                }
              }
            }
          }
        }
      }
    }
  },
  "throttle_period": "30m", 
  "condition": {
    "script":  "if (ctx.payload.aggregations.minutes.buckets.size() == 0) return false; def latest = ctx.payload.aggregations.minutes.buckets[-1]; def node = latest.nodes.buckets[0]; return node && node.cpu && node.cpu.value >= 75;"
  },
  "actions": {
    "send_email": { 
      "transform": {
        "script": "def latest = ctx.payload.aggregations.minutes.buckets[-1]; return latest.nodes.buckets.findAll { return it.cpu && it.cpu.value >= 75 };"
      },
      "email": {
        "to": "user@example.com", 
        "subject": "Watcher Notification - HIGH CPU USAGE",
        "body": "Nodes with HIGH CPU Usage (above 75%):\n\n{{#ctx.payload._value}}\"{{key}}\" - CPU Usage is at {{cpu.value}}%\n{{/ctx.payload._value}}"
      }
    }
  }
}

 4.监控文件目录
每分钟检测一次,如果节点目录使用率大于80%,则发送预警邮件

PUT _watcher/watch/open_file_descriptors

{
  "metadata": {
    "system_fd": 65535,
    "threshold": 0.8
  },
  "trigger": {
    "schedule": {
      "interval": "1m"
    }
  },
  "input": {
    "search": {
      "request": {
        "indices": [
          ".marvel-*"
        ],
        "types": "node_stats",
        "search_type": "count",
        "body": {
          "query": {
            "filtered": {
              "filter": {
                "range": {
                  "@timestamp": {
                    "gte": "now-1m",
                    "lte": "now"
                  }
                }
              }
            }
          },
          "aggs": {
            "minutes": {
              "date_histogram": {
                "field": "@timestamp",
                "interval": "5s"
              },
              "aggs": {
                "nodes": {
                  "terms": {
                    "field": "node.name.raw",
                    "size": 10,
                    "order": {
                      "fd": "desc"
                    }
                  },
                  "aggs": {
                    "fd": {
                      "avg": {
                        "field": "process.open_file_descriptors"
                      }
                    }
                  }
                }
              }
            }
          }
        }
      }
    }
  },
  "throttle_period": "30m", 
  "condition": {
    "script": "if (ctx.payload.aggregations.minutes.buckets.size() == 0) return false; def latest = ctx.payload.aggregations.minutes.buckets[-1]; def node = latest.nodes.buckets[0]; return node && node.fd && node.fd.value >= (ctx.metadata.system_fd * ctx.metadata.threshold);"
  },
  "actions": {
    "send_email": { 
      "transform": {
        "script": "def latest = ctx.payload.aggregations.minutes.buckets[-1]; return latest.nodes.buckets.findAll({ return it.fd && it.fd.value >= (ctx.metadata.system_fd * ctx.metadata.threshold) }).collect({ it.fd.percent = Math.round((it.fd.value/ctx.metadata.system_fd)*100); it });"
      },
      "email": {
        "to": "<username>@<domainname>", 
        "subject": "Watcher Notification - NODES WITH 80% FILE DESCRIPTORS USED",
        "body": "Nodes with 80% FILE DESCRIPTORS USED (above 80%):\n\n{{#ctx.payload._value}}\"{{key}}\" - File Descriptors is at {{fd.value}} ({{fd.percent}}%)\n{{/ctx.payload._value}}"
      }
    }
  }
}

 5.监控Field 缓存相关
每分钟执行一次,如果使用率大于80%,则发送预警邮件。

PUT _watcher/watch/fielddata_utilization
{
  "metadata": {
    "fielddata_cache_size": 100000, 
    "threshold": 0.8
  },
  "trigger": {
    "schedule": {
      "interval": "1m"
    }
  },
  "input": {
    "search": {
      "request": {
        "indices": [
          ".marvel-*"
        ],
        "types": "node_stats",
        "search_type": "count",
        "body": {
          "query": {
            "filtered": {
              "filter": {
                "range": {
                  "@timestamp": {
                    "gte": "now-1m",
                    "lte": "now"
                  }
                }
              }
            }
          },
          "aggs": {
            "minutes": {
              "date_histogram": {
                "field": "@timestamp",
                "interval": "5s"
              },
              "aggs": {
                "nodes": {
                  "terms": {
                    "field": "node.name.raw",
                    "size": 10,
                    "order": {
                      "fielddata": "desc"
                    }
                  },
                  "aggs": {
                    "fielddata": {
                      "avg": {
                        "field": "indices.fielddata.memory_size_in_bytes"
                      }
                    }
                  }
                }
              }
            }
          }
        }
      }
    }
  },
  "throttle_period": "30m", 
  "condition": {
    "script": "if (ctx.payload.aggregations.minutes.buckets.size() == 0) return false; def latest = ctx.payload.aggregations.minutes.buckets[-1]; def node = latest.nodes.buckets[0]; return node && node.fielddata && node.fielddata.value >= (ctx.metadata.fielddata_cache_size * ctx.metadata.threshold);"
  },
  "actions": {
    "send_email": { 
      "transform": {
        "script": "def latest = ctx.payload.aggregations.minutes.buckets[-1]; return latest.nodes.buckets.findAll({ return it.fielddata && it.fielddata.value >= (ctx.metadata.fielddata_cache_size * ctx.metadata.threshold) }).collect({ it.fielddata.percent = Math.round((it.fielddata.value/ctx.metadata.fielddata_cache_size)*100); it });"
      },
      "email": {
        "to": "<username>@<domainname>", 
        "subject": "Watcher Notification - NODES WITH 80% FIELDDATA UTILIZATION",
        "body": "Nodes with 80% FIELDDATA UTILIZATION (above 80%):\n\n{{#ctx.payload._value}}\"{{key}}\" - Fielddata utilization is at {{fielddata.value}} bytes ({{fielddata.percent}}%)\n{{/ctx.payload._value}}"
      }
    }
  }
}

 6.监控集群节点
每分钟执行一次,如果有Node加入或者离开集群,则发送预警邮件。

PUT _watcher/watch/node_event
{
  "trigger": {
    "schedule": {
      "interval": "60s"
    }
  },
  "input": {
    "search": {
      "request": {
        "indices": [
          ".marvel-*"
        ],
        "search_type": "query_then_fetch",
        "body": {
          "query": {
            "filtered": {
              "query": {
                "bool": {
                  "should": [
                    {
                      "match": {
                        "event": "node_left"
                      }
                    },
                    {
                      "match": {
                        "event": "node_joined"
                      }
                    }
                  ]
                }
              },
              "filter": {
                "range": {
                  "@timestamp": {
                    "from": "{{ctx.trigger.scheduled_time}}||-60s",
                    "to": "{{ctx.trigger.triggered_time}}"
                  }
                }
              }
            }
          },
          "fields": [
            "event",
            "message",
            "cluster_name"
          ],
          "sort": [
            {
              "@timestamp": {
                "order": "desc"
              }
            }
          ]
        }
      }
    }
  },
  "throttle_period": "60s", 
  "condition": {
    "script": {
      "inline": "ctx.payload.hits.size() > 0 "
    }
  },
  "actions": {
    "send_email": { 
      "email": {
        "to": "<username>@<domainname>", 
        "subject": "{{ctx.payload.hits.hits.0.fields.event}} the cluster",
        "body": "{{ctx.payload.hits.hits.0.fields.message}} the cluster {{ctx.payload.hits.hits.0.fields.cluster_name}} "
      }
    }
  }
}

 

评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值