背景:由于alertmanager没有历史告警存储功能。在工作中我们可能需要 收集历史告警信息进行数据分析,找到系统的瓶颈,从而提高稳定性
实现方法:go语言写一个 针对alertmanager的webhook 程序。收集告警信息进行过滤然后存入es中
具体代码如下
import (
"bytes"
"context"
"encoding/json"
"fmt"
"github.com/elastic/go-elasticsearch/v8"
"github.com/elastic/go-elasticsearch/v8/esapi"
"github.com/gin-gonic/gin"
"log"
"net/http"
"reflect"
"time"
)
type Alert struct {
Status string `json:"status"`
Labels Labels `json:"labels"`
Annotations Annotations `json:"annotations"`
StartsAt string `json:"startsAt"`
EndsAt string `json:"endsAt"`
//GeneratorURL string `json:"generatorURL"`
//Fingerprint string `json:"fingerprint"`
}
type Labels struct {
AlertName string `json:"alertname"`
Group string `json:"group"`
Instance string `json:"instance"`
Job string `json:"job"`
Severity string `json:"severity"`
Team string `json:"team"`
}
type Annotations struct {
Description string `json:"description"`
//Summary string `json:"summary"`
//Value string `json:"value"`
}
type AlertDetail struct {
AlertName string
Group string
Severity string
Team string
Description string
Time float64
}
func HandleJSON(c *gin.Context) {
var data struct {
//Receiver string `json:"receiver"`
Status string `json:"status"`
Alerts []Alert `json:"alerts"`
//GroupLabels Labels `json:"groupLabels"`
//CommonLabels Labels `json:"commonLabels"`
//CommonAnnotations Annotations `json:"commonAnnotations"`
//TruncatedAlerts int `json:"truncatedAlerts"`
}
if err := c.ShouldBindJSON(&data); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
//fmt.Println(data)
if data.Status != "resolved" {
c.JSON(http.StatusOK, gin.H{"message": "JSON data received and processed"})
return
}
//配置es
cfg := elasticsearch.Config{
Addresses: []string{"http://127.0.0.1:9200"}, // 连接地址,根据自己的实际情况修改
}
// 创建 Elasticsearch 客户端
client, err := elasticsearch.NewClient(cfg)
if err != nil {
fmt.Println("Error creating the client: ", err)
return
}
for _, a := range data.Alerts {
startsAt, _ := time.Parse(time.RFC3339, a.StartsAt)
endsAt, _ := time.Parse(time.RFC3339, a.EndsAt)
duration := endsAt.Sub(startsAt)
durationInHours := duration.Hours()
alertdetail := AlertDetail{
AlertName: a.Labels.AlertName,
Group: a.Labels.Group,
Team: a.Labels.Team,
Description: a.Annotations.Description,
Time: durationInHours,
}
fmt.Println(alertdetail)
// 将文档结构体转换为 JSON 字符串
jsonStr, err := json.Marshal(alertdetail)
if err != nil {
fmt.Println("Error marshaling document: ", err)
return
}
// 创建索引请求
req := esapi.IndexRequest{
Index: "alert",
Body: bytes.NewReader(jsonStr),
}
// 执行索引请求
res, err := req.Do(context.Background(), client)
if err != nil {
fmt.Println("Error executing the request: ", err)
return
}
defer res.Body.Close()
// 检查响应状态码
if res.IsError() {
fmt.Printf("Error indexing document: %s", res.Status())
} else {
fmt.Println("Document indexed successfully")
}
}
c.JSON(http.StatusOK, gin.H{"message": "JSON data received and processed"})
}
func SearchES(c *gin.Context) {
//配置es
cfg := elasticsearch.Config{
Addresses: []string{"http://127.0.0.1:9200"}, // 连接地址,根据自己的实际情况修改
}
// 创建 Elasticsearch 客户端
client, err := elasticsearch.NewClient(cfg)
if err != nil {
fmt.Println("Error creating the client: ", err)
return
}
//获取es数据
// 构造查询请求
req := esapi.SearchRequest{
Index: []string{"alert"}, // 索引名称
}
res, err := req.Do(context.Background(), client)
if err != nil {
log.Fatalf("Error executing search request: %s", err)
}
defer res.Body.Close()
// 处理响应结果
if res.IsError() {
log.Fatalf("Search request returned error code %d", res.StatusCode)
}
respjson := res.String()
fmt.Println(respjson)
fmt.Println(reflect.TypeOf(respjson))
c.JSON(http.StatusOK, respjson)
}
func main() {
router := gin.Default()
router.POST("/receive", HandleJSON)
router.GET("/search", SearchES)
// 启动服务器
router.Run(":8080")
}