1.安装Grafana
wget https://dl.grafana.com/oss/release/grafana-6.4.3-1.x86_64.rpm
yum install localhost grafana-6.4.3-1.x86_64.rpm
service grafana-server start
安装饼图插件
grafana-cli plugins install grafana-piechart-panel
service grafana-server restart
2.安装Prometheus
2.1 下载prometheus
wget https://github.com/prometheus/prometheus/releases/download/v2.13.1/prometheus-2.13.1.linux-amd64.tar.gz
tar -zxvf prometheus-2.13.1.linux-amd64.tar.gz -C /usr/local
cd /usr/local/prometheus-2.13.1.linux-amd64/
2.编辑配置文件
vim prometheus.yml
# my global config
global:
scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
# scrape_timeout is set to the global default (10s).
# Alertmanager configuration
alerting:
alertmanagers:
- static_configs:
- targets: ['localhost:9093']
# - alertmanager:9093
# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
rule_files:
- "slave_status-rules.yml"
#- "sql_rules.yml"
# - "second_rules.yml"
remote_write:
- url: "http://192.168.66.118:8086/api/v1/prom/write?db=prometheus"
remote_read:
- url: "http://192.168.66.118:8086/api/v1/prom/read?db=prometheus"
# A scrape configuration containing exactly one endpoint to scrape:
# Here it's Prometheus itself.
scrape_configs:
# The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
- job_name: 'prometheus'
# metrics_path defaults to '/metrics'
# scheme defaults to 'http'.
static_configs:
- targets: ['localhost:9090']
# 以下是添加的部分
- job_name: 'mysql'
file_sd_configs:
- files:
- targets/mysql/fangzhen/*.json
- targets/mysql/produce/*.json
refresh_interval: 1m
3.启动prometheus
mkdir -p /data/prometheus/var
nohup ./prometheus --storage.tsdb.path="/data/prometheus/var/" --config.file="prometheus.yml" 2>&1 1> nohup.out &
3、安装prometheus-webhook-dingtal
3.1直接安装bin包
https://github.com/timonwong/prometheus-webhook-dingtalk/releases
下载prometheus-webhook-dingtalk-1.4.0.linux-amd64.tar.gz
tar -zxvf prometheus-webhook-dingtalk-1.4.0.linux-amd64.tar.gz
3.2编译安装
3.2.1解决依赖
(1)安装go环境
tar -xvf go1.13.linux-amd64.tar.gz -C /usr/local/
cat >> /etc/profile << EOF
export PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/go/bin
export GOPATH=/usr/local/gocode
export GOROOT=/usr/local/go/bin
EOF
source /etc/profile
go version
(2)yarn
curl --silent --location https://dl.yarnpkg.com/rpm/yarn.repo | sudo tee /etc/yum.repos.d/yarn.repo
curl --silent --location https://rpm.nodesource.com/setup_8.x | sudo bash -
yum install yarn
(3)node.js 版本
node.js 要求版本大于等于8
wget https://nodejs.org/dist/v12.16.1/node-v12.16.1-linux-x64.tar.xz
下载node.js 新版本
tar xf node-v12.16.1-linux-x64.tar.xz -C /usr/local/
mv /usr/bin/node /usr/bin/node_bakv6.17.1
ln -s /usr/local/node-v12.16.1-linux-x64/bin/node /usr/bin/node
(4)gcc在4.8.2版本以上
cd gcc-4.8.2
./contrib/download_prerequisites
mkdir gcc-build-4.8.2
cd gcc-build-4.8.2
../configure -enable-checking=release -enable-languages=c,c++ -disable-multilib
make -j 32
sudo make install
3.2.2.编译prometheus-webhook-dingtal
cd /usr/local/src/
git clone https://github.com/timonwong/prometheus-webhook-dingtalk.git
cd prometheus-webhook-dingtalk
make build
3.3编辑模版文件
vim tmpl/DBA.tmpl
{{ define "__subject" }}[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .GroupLabels.SortedPairs.Values | join " " }} {{ if gt (len .CommonLabels) (len .GroupLabels) }}({{ with .CommonLabels.Remove .GroupLabels.Names }}{{ .Values | join " " }}{{ end }}){{ end }}{{ end }}
{{ define "__alertmanagerURL" }}{{ .ExternalURL }}/#/alerts?receiver={{ .Receiver }}{{ end }}
{{ define "__text_alert_list" }}{{ range . }}
【触发时间】: {{ .StartsAt.Format "2006-01-02 15:04:05" }}
{{ range .Labels.SortedPairs }}
{{ if eq .Name "pro_name" }}> - 【所属项目】: {{ .Value }}{{ end }}
{{ if eq .Name "cls_id" }}> - 【集群id】: {{ .Value }}{{ end }}
{{ if eq .Name "instance" }}> - 【故障主机】: {{ .Value }}{{ end }}
{{ end }}
{{ range .Annotations.SortedPairs }}
{{ if eq .Name "summary" }}> - 【故障描述】: {{ .Value | markdown | html }}{{ end }}
{{ end }}
{{ end }}{{ end }}
{{ define "__text_resolved_list" }}{{ range . }}
【触发时间】: {{ .StartsAt.Format "2006-01-02 15:04:05" }}
【恢复时间】: {{ .EndsAt.Format "2006-01-02 15:04:05" }}
{{ range .Labels.SortedPairs }}
{{ if eq .Name "pro_name" }}> - 【所属项目】: {{ .Value }}{{ end }}
{{ if eq .Name "cls_id" }}> - 【集群id】: {{ .Value }}{{ end }}
{{ if eq .Name "instance" }}> - 【故障主机】: {{ .Value }}{{ end }}
{{ end }}
{{ range .Annotations.SortedPairs }}
{{ if eq .Name "summary" }}> - 【故障描述】: {{ .Value | markdown | html }}{{ end }}
{{ end }}
{{ end }}{{ end }}
{{ define "ding.link.title" }}{{ template "__subject" . }}{{ end }}
{{ define "ding.link.content" }}
{{ if gt (len .Alerts.Firing) 0 }}#### [{{ .Alerts.Firing | len }}]【Firing】
{{ template "__text_alert_list" .Alerts.Firing }}{{ end }}
{{ if gt (len .Alerts.Resolved) 0 }}#### [{{ .Alerts.Resolved | len }}]【Resolved】
{{ template "__text_resolved_list" .Alerts.Resolved }}{{ end }}
{{ end }}
4、启动prometheus-webhook-dingtal
nohup ./prometheus-webhook-dingtalk --config.file=./config.yml 2>&1 1>out.log &
四、安装alertmanager
1.下载并解压
wget https://github.com/prometheus/alertmanager/releases/download/v0.19.0/alertmanager-0.19.0.linux-amd64.tar.gz
tar -zxvf alertmanager-0.19.0.linux-amd64.tar.gz -C /usr/local
cd /usr/local/alertmanager-0.19.0.linux-amd64
2.编辑配置文件
vim alertmanager.yml
global:
resolve_timeout: 2m
route:
receiver: dingding
group_wait: 30s
group_interval: 5m
repeat_interval: 4h
group_by: [alertname]
routes:
- receiver: dingding
group_wait: 10s
match:
team: node
receivers:
- name: dingding
webhook_configs:
- send_resolved: true
url: http://localhost:8060/dingtalk/ops_dingding/send
3.启动alertmanager
nohup ./alertmanager --config.file=alertmanager.yml 2>&1 1>out.log &