prometheus告警规则从数据库读写

1 prometheus源码安装

proemtheus源码安装需要node.js yarn go环境

1.1 下载go环境

wget https://golang.google.cn/dl/go1.15.linux-amd64.tar.gz
tar -xzvf go1.15.linux-amd64.tar.gz -C /usr/local/
mkdir /go_code
vim /etc/profile
export GOROOT=/usr/local/go/
export PATH=$PATH:$GOROOT/bin
export GOPATH=/go_code
source /etc/profile
go version
go version go1.15 linux/amd64

1.2 安装node.js

curl --silent --location https://rpm.nodesource.com/setup_10.x | sudo bash -
yum install nodejs
node -v
v10.22.0

1.3 安装yarn

curl --silent --location https://dl.yarnpkg.com/rpm/yarn.repo | sudo tee /etc/yum.repos.d/yarn.repo
sudo rpm --import https://dl.yarnpkg.com/rpm/pubkey.gpg
sudo yum install yarn
yarn --version
1.22.4

1.4 clone prometheus源码

mkdir -p $GOPATH/src/github.com/prometheus
cd $GOPATH/src/github.com/prometheus
git clone https://github.com/prometheus/prometheus.git
cd prometheus
make build  #make build会生成二进制可执行文件
./prometheus --config.file=your_config.yml

2 mysql表设计

2.1 创建prometheus数据库

MariaDB [(none)]> create database prometheus;

2.2 创建数据库表

CREATE TABLE rules(
rule_id INT UNSIGNED NOT NULL auto_increment primary key,
rule_name VARCHAR(100),
rule_fn VARCHAR(100),
rule_interval INT UNSIGNED NOT NULL,
rule_alert VARCHAR(100),
rule_expr VARCHAR(100),
rule_for VARCHAR(100),
rule_labels VARCHAR(100),
rule_annotations VARCHAR(100)
);

3 修改promethes源码

3.1 加载mysql模块

prometheus目录结构如下：

3.1.1 下载go操作数据库驱动包

cd $GOPATH/src/github.com
go clone https://github.com/go-sql-driver/mysql.git

3.1.2 加载MySQL驱动

// models/mysql.go
package models

import (
    "database/sql"
    _ "github.com/go-sql-driver/mysql"
    "container/list"
    "log"
    "strings"
)
var (
    _SQL_DB *sql.DB
    _ERR error
)

//规则结构定义
type RuleItem struct {
    Name        string            //规则所属组得名称
    Fn          string            //类别
    Interval    int               //规则计算间隔
    Alert       string            //告警名称
    Expr        string            //规则表达式
    For         string            //持续时间
    Labels      map[string]string //规则维度信息
    Annotations map[string]string //规则描述信息
}
// 初始化数据库连接
func Initialzation(dbUrl string){
    _SQL_DB, _ERR = sql.Open("mysql", dbUrl)
    if _ERR != nil{
        log.Fatalf("Open database error: %s\n", _ERR)
        return
    }
    _ERR = _SQL_DB.Ping()
    if _ERR != nil{
        log.Fatal(_ERR)
    }
}
//规则查询,用于将rules表转化为RuleItem结构。
func QueryRuleString()(*list.List, error){
    var(
        rule_labels,rule_annotations string
    )
    l := list.New()
    // 查询规则表
    rows, err := _SQL_DB.Query("select rule_name,rule_fn,rule_interval,rule_alert,rule_expr,rule_for,rule_labels,rule_annotations from rules;")
    if err != nil{
        log.Println(err)
    }
    defer rows.Close()
    for rows.Next(){
        var item RuleItem
        item.Labels=make(map[string]string)
        item.Annotations=make(map[string]string)
        err := rows.Scan(&item.Name,&item.Fn,&item.Interval,&item.Alert,&item.Expr,&item.For,&rule_labels,&rule_annotations)
        if err != nil{
            log.Fatal(err)
        }
        //label 数据格式转换
        labels := strings.Split(rule_labels, ",")
        lablen := len(labels)
        for i:=0;i<lablen;i++ {
            pars := strings.Split(labels[i], "=")
            plen := len(pars)
            for j:=0;j<plen;j+=2 {
                item.Labels[pars[j]]=pars[j+1]
            }
        }
        //annotations数据格式转换
        annotations := strings.Split(rule_annotations, ",")
        annlen := len(annotations)
        for k:=0;k<annlen;k++{
            pars := strings.Split(annotations[k], "=")
            plen := len(pars)
            for j:=0;j<plen;j+=2{
                item.Annotations[pars[j]]=pars[j+1]
            }
        }
        l.PushBack(item)
    }
    return l,err
}

3.2 修改读取告警规则配置

//rules/manager.go
import(
"github.com/prometheus/prometheus/models"
)
// LoadGroups reads groups from a list of files.
func (m *Manager) LoadGroups(
    interval time.Duration, externalLabels labels.Labels, filenames ...string,
) (map[string]*Group, []error) {
    groups := make(map[string]*Group)
    shouldRestore := !m.restored
// 添加
    var rName,rFn string
    rulelist,rerr := models.QueryRuleString()
    if rerr != nil{
        return nil, []error{rerr}
    }
    itv := interval
    rules := make([]Rule, 0, rulelist.Len())
    for e := rulelist.Front(); e != nil; e = e.Next(){
        r := (e.Value).(models.RuleItem)
        rName = r.Name
        rFn = r.Fn
        expr, err := parser.ParseExpr(r.Expr)
        if err != nil {
            return nil, []error{err}
        }
        if r.Interval != 0{
            itv = time.Duration(r.Interval)
        }
        dur, derr := model.ParseDuration(r.For)
        if derr != nil {
            return nil, []error{err}
        }
        //构建AlertingRule实例并将其添加到rules中
        rules = append(rules, NewAlertingRule(
            r.Alert,
            expr,
            time.Duration(dur),
            labels.FromMap(r.Labels),
            labels.FromMap(r.Annotations),
            externalLabels,
            m.restored,
            log.With(m.logger, "alert", r.Alert),
        ))
    }
    if len(rules) > 0 {
        groups[groupKey(rName, rFn)] = NewGroup(GroupOptions{
            Name: rName,
            File: rFn,
            Interval: itv,
            Rules: rules,
            ShouldRestore: shouldRestore,
            Opts: m.opts,
            done: m.done,
            })
    }
    //结束
    for _, fn := range filenames {
        rgs, errs := m.opts.GroupLoader.Load(fn)
        if errs != nil {
            return nil, errs
        }

        for _, rg := range rgs.Groups {
            itv := interval
            if rg.Interval != 0 {
                itv = time.Duration(rg.Interval)
            }

            rules := make([]Rule, 0, len(rg.Rules))
            for _, r := range rg.Rules {
                expr, err := m.opts.GroupLoader.Parse(r.Expr.Value)
                if err != nil {
                    return nil, []error{errors.Wrap(err, fn)}
                }

                if r.Alert.Value != "" {
                    rules = append(rules, NewAlertingRule(
                        r.Alert.Value,
                        expr,
                        time.Duration(r.For),
                        labels.FromMap(r.Labels),
                        labels.FromMap(r.Annotations),
                        externalLabels,
                        m.restored,
                        log.With(m.logger, "alert", r.Alert),
                    ))
                    continue
                }
                rules = append(rules, NewRecordingRule(
                    r.Record.Value,
                    expr,
                    labels.FromMap(r.Labels),
                ))
            }
            groups[groupKey(fn, rg.Name)] = NewGroup(GroupOptions{
                Name:          rg.Name,
                File:          fn,
                Interval:      itv,
                Rules:         rules,
                ShouldRestore: shouldRestore,
                Opts:          m.opts,
                done:          m.done,
            })
        }
    }
    return groups, nil
}

3.3 修改main.go连接数据库

// cmd/prometheus/main.go
func main() {
//再添加全局配置处，添加数据库连接url配置
	cfg := struct {
		configFile string
		dbUrl	string
		}
	a.Flag("config.file", "Prometheus configuration file path.").
		Default("prometheus.yml").StringVar(&cfg.configFile)
	a.Flag("mysql.url", "Mysql connection path.").
		Default("root:123456@tcp(localhost:3306)/prometheus").StringVar(&cfg.dbUrl)
	logger := promlog.New(&cfg.promlogConfig)
	//连接数据库
	models.Initialzation(cfg.dbUrl)
}

3.4 编译二进制包

cd /go_code/src/github.com/prometheus/prometheus
make build
-rwxr-xr-x  1 root root 88101796 Aug 18 11:26 prometheus //生成文件

3.5 制作docker镜像

make -p .build/linux-amd64
cp prometheus .build/linux-amd64
cp promtool .build/linux-amd64
make npm_licenses
docker build -t prometheus-mysql .
生成得docker镜像
docker images
REPOSITORY                               TAG                 IMAGE ID            CREATED             SIZE
prometheus-mysql                         latest              ea7921aa11fc        4 days ago          140MB
docker镜像启动
docker run -d --restart=always --name prometheus --net host \
-v /opt/prometheus/:/etc/prometheus/ -v /etc/localtime:/etc/localtime:ro \
-v /opt/prometheus-data:/prometheus prometheus-mysql-db \
--config.file=/etc/prometheus/prometheus.yml \
--mysql.url="root:123456@tcp(localhost:3306)/prometheus" \
--log.level=info --web.enable-lifecycle --web.enable-admin-api \
--web.listen-address="0.0.0.0:9090" --storage.tsdb.path=/prometheus \
--storage.tsdb.max-block-duration=2h --storage.tsdb.min-block-duration=2h \
--storage.tsdb.wal-compression --storage.tsdb.retention.time=2h

4 mysql读写测试

4.1 插入数据

insert into rules(rule_id,rule_name,rule_fn,rule_interval,rule_alert,rule_expr,rule_for,rule_labels,rule_annotations) value('1','prometheus','alert','30s','test1','up{job="prometheus"}==0','30s','severity=page','summary={{ $labels.instance }} of job {{ $labels.job }} has been down for more than 30 seconds');

4.2 prometheus发送reload请求

curl -X POST http://${ip}:9090/-/reload

4.3 查看报警规则