Prometheus+Grafana监控部署

1.安装Grafana

wget https://dl.grafana.com/oss/release/grafana-6.4.3-1.x86_64.rpm
yum install localhost grafana-6.4.3-1.x86_64.rpm
service grafana-server start

安装饼图插件

 grafana-cli plugins install grafana-piechart-panel
 service grafana-server restart

2.安装Prometheus

2.1 下载prometheus

 wget https://github.com/prometheus/prometheus/releases/download/v2.13.1/prometheus-2.13.1.linux-amd64.tar.gz
 tar -zxvf prometheus-2.13.1.linux-amd64.tar.gz -C /usr/local
 cd  /usr/local/prometheus-2.13.1.linux-amd64/

2.编辑配置文件

vim prometheus.yml

# my global config
global:
 scrape_interval:     15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
 evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
 # scrape_timeout is set to the global default (10s).

# Alertmanager configuration
alerting:
 alertmanagers:
 - static_configs:
   - targets: ['localhost:9093']
     # - alertmanager:9093

# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
rule_files:
 - "slave_status-rules.yml"
 #- "sql_rules.yml"
 # - "second_rules.yml"
remote_write:
 - url: "http://192.168.66.118:8086/api/v1/prom/write?db=prometheus"
remote_read:
 - url: "http://192.168.66.118:8086/api/v1/prom/read?db=prometheus"

# A scrape configuration containing exactly one endpoint to scrape:
# Here it's Prometheus itself.
scrape_configs:
 # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
 - job_name: 'prometheus'

   # metrics_path defaults to '/metrics'
   # scheme defaults to 'http'.

   static_configs:
   - targets: ['localhost:9090']

# 以下是添加的部分
 - job_name: 'mysql'
   file_sd_configs:
     - files:
       - targets/mysql/fangzhen/*.json
       - targets/mysql/produce/*.json
       refresh_interval: 1m

3.启动prometheus

mkdir -p /data/prometheus/var
nohup ./prometheus --storage.tsdb.path="/data/prometheus/var/" --config.file="prometheus.yml" 2>&1 1> nohup.out &

3、安装prometheus-webhook-dingtal

3.1直接安装bin包

https://github.com/timonwong/prometheus-webhook-dingtalk/releases
下载prometheus-webhook-dingtalk-1.4.0.linux-amd64.tar.gz

 tar -zxvf prometheus-webhook-dingtalk-1.4.0.linux-amd64.tar.gz

3.2编译安装

3.2.1解决依赖

（1）安装go环境

tar -xvf go1.13.linux-amd64.tar.gz -C /usr/local/
cat >>  /etc/profile  << EOF
export PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/go/bin
export GOPATH=/usr/local/gocode
export GOROOT=/usr/local/go/bin
EOF
source /etc/profile
go version

（2）yarn

curl --silent --location https://dl.yarnpkg.com/rpm/yarn.repo | sudo tee /etc/yum.repos.d/yarn.repo
curl --silent --location https://rpm.nodesource.com/setup_8.x | sudo bash -
yum install yarn

（3）node.js 版本
node.js 要求版本大于等于8

wget https://nodejs.org/dist/v12.16.1/node-v12.16.1-linux-x64.tar.xz
下载node.js 新版本
tar xf node-v12.16.1-linux-x64.tar.xz -C /usr/local/
mv /usr/bin/node /usr/bin/node_bakv6.17.1
ln -s /usr/local/node-v12.16.1-linux-x64/bin/node /usr/bin/node

（4）gcc在4.8.2版本以上

cd gcc-4.8.2　
./contrib/download_prerequisites　
mkdir gcc-build-4.8.2
cd gcc-build-4.8.2
../configure -enable-checking=release -enable-languages=c,c++ -disable-multilib
make -j 32
sudo make install

3.2.2.编译prometheus-webhook-dingtal

cd /usr/local/src/
git clone https://github.com/timonwong/prometheus-webhook-dingtalk.git
cd prometheus-webhook-dingtalk
make build

3.3编辑模版文件

vim tmpl/DBA.tmpl

{{ define "__subject" }}[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .GroupLabels.SortedPairs.Values | join " " }} {{ if gt (len .CommonLabels) (len .GroupLabels) }}({{ with .CommonLabels.Remove .GroupLabels.Names }}{{ .Values | join " " }}{{ end }}){{ end }}{{ end }}
{{ define "__alertmanagerURL" }}{{ .ExternalURL }}/#/alerts?receiver={{ .Receiver }}{{ end }}

{{ define "__text_alert_list" }}{{ range . }}
【触发时间】: {{ .StartsAt.Format "2006-01-02 15:04:05" }}
{{ range .Labels.SortedPairs }}
{{ if eq .Name "pro_name" }}> - 【所属项目】: {{ .Value }}{{ end }}
{{ if eq .Name "cls_id" }}> - 【集群id】: {{ .Value }}{{ end }}
{{ if eq .Name "instance" }}> - 【故障主机】: {{ .Value }}{{ end }}
{{ end }}
{{ range .Annotations.SortedPairs }}
{{ if eq .Name "summary" }}> - 【故障描述】: {{ .Value | markdown | html }}{{ end }}
{{ end }}
{{ end }}{{ end }}

{{ define "__text_resolved_list" }}{{ range . }}
【触发时间】: {{ .StartsAt.Format "2006-01-02 15:04:05" }}
【恢复时间】: {{ .EndsAt.Format "2006-01-02 15:04:05" }}
{{ range .Labels.SortedPairs }}
{{ if eq .Name "pro_name" }}> - 【所属项目】: {{ .Value }}{{ end }}
{{ if eq .Name "cls_id" }}> - 【集群id】: {{ .Value }}{{ end }}
{{ if eq .Name "instance" }}> - 【故障主机】: {{ .Value }}{{ end }}
{{ end }}
{{ range .Annotations.SortedPairs }}
{{ if eq .Name "summary" }}> - 【故障描述】: {{ .Value | markdown | html }}{{ end }}
{{ end }}
{{ end }}{{ end }}


{{ define "ding.link.title" }}{{ template "__subject" . }}{{ end }}
{{ define "ding.link.content" }}
{{ if gt (len .Alerts.Firing) 0 }}#### [{{ .Alerts.Firing | len }}]【Firing】
{{ template "__text_alert_list" .Alerts.Firing }}{{ end }}
{{ if gt (len .Alerts.Resolved) 0 }}#### [{{ .Alerts.Resolved | len }}]【Resolved】
{{ template "__text_resolved_list" .Alerts.Resolved }}{{ end }}
{{ end }}

4、启动prometheus-webhook-dingtal

 nohup  ./prometheus-webhook-dingtalk --config.file=./config.yml    2>&1 1>out.log &

四、安装alertmanager

1.下载并解压

 wget https://github.com/prometheus/alertmanager/releases/download/v0.19.0/alertmanager-0.19.0.linux-amd64.tar.gz
 tar -zxvf alertmanager-0.19.0.linux-amd64.tar.gz -C /usr/local
 cd /usr/local/alertmanager-0.19.0.linux-amd64

2.编辑配置文件

vim alertmanager.yml

global:
resolve_timeout: 2m
route:
receiver: dingding
group_wait: 30s
group_interval: 5m
repeat_interval: 4h
group_by: [alertname]
routes:
- receiver: dingding
  group_wait: 10s
  match:
    team: node
receivers:
- name: dingding
webhook_configs:
- send_resolved: true
  url: http://localhost:8060/dingtalk/ops_dingding/send 

3.启动alertmanager

 nohup ./alertmanager  --config.file=alertmanager.yml 2>&1 1>out.log &