saltstack部署Prometheus

⽤saltstack编写普罗米修斯状态文件

项目结构

说明环境

主机ip	主机名	安装应用
192.168.129.135	server	prometheus grafana
192.168.129.136	agent	node_exporter

安装包
prometheus-2.31.1.linux-amd64.tar.gz
node_exporter-1.3.0.linux-amd64.tar.gz
grafana-enterprise-8.2.5-1.x86_64.rpm

[root@master ~]# cd /srv/salt/prod/modules
[root@master modules]# tree prometheus/
prometheus/
├── agent.sls
├── files
│   ├── grafana-enterprise-8.2.5-1.x86_64.rpm
│   ├── node_exporter-1.3.0.linux-amd64.tar.gz
│   ├── node_exporter.service.j2
│   ├── prometheus-2.31.1.linux-amd64.tar.gz
│   ├── prometheus.service.j2
│   └── prometheus.yml.j2
└── server.sls

1 directory, 8 files

pillar变量

[root@master prod]# pwd
/srv/pillar/prod
[root@master prod]# cat prometheus.sls 
prometheus_install_dir: /usr/local
nodes:
  - 192.168.129.133:9100
  - 192.168.129.136:9100
  - 192.168.129.137:9100
node_exporter_install_dir: /usr/local/

[root@master prod]# cat top.sls 
prod:
  '*':
    - prometheus

prometheus以及grafana状态文件

[root@master prometheus]# pwd
/srv/salt/prod/modules/prometheus

[root@master prometheus]# cat server.sls 
copy-grafana-enterprise:
  file.managed:
    - name: /usr/src/grafana-enterprise-8.2.5-1.x86_64.rpm
    - source: salt://modules/prometheus/files/grafana-enterprise-8.2.5-1.x86_64.rpm

install-grafana:
  cmd.run:
    - name: "cd /usr/src/ && yum -y install grafana-enterprise-8.2.5-1.x86_64.rpm"
    - unless: test $(rpm -qa | grep grafana-enterprise-8.2.5-1.x86_64)

unzip-prometheus:
  archive.extracted:
    - name: {{ pillar['prometheus_install_dir'] }}
    - source: salt://modules/prometheus/files/prometheus-2.31.1.linux-amd64.tar.gz
    - unless: test -d {{ pillar['prometheus_install_dir'] }}/prometheus-2.31.1.linux-amd64
  file.symlink:
    - name: {{ pillar['prometheus_install_dir'] }}/prometheus
    - target: {{ pillar['prometheus_install_dir'] }}/prometheus-2.31.1.linux-amd64

copy-files:
  file.managed:
    - names:
      - /usr/lib/systemd/system/prometheus.service:
        - source: salt://modules/prometheus/files/prometheus.service.j2
        - template: jinja
      - {{ pillar['prometheus_install_dir'] }}/prometheus/prometheus.yml:
        - source: salt://modules/prometheus/files/prometheus.yml.j2
        - template: jinja

start-grafana-enterprise:
  service.running:
    - name: grafana-server.service
    - enable: true
    - reload: true

start-prometheus:
  service.running:
    - name: prometheus.service
    - enable: true
    - reload: true
    - watch:
      - file: {{ pillar['prometheus_install_dir'] }}/prometheus/prometheus.yml

prometheus.yml.j2 配置文件

[root@master files]# pwd
/srv/salt/prod/modules/prometheus/files
[root@master prometheus]# cat files/prometheus.yml.j2 
# my global config
global:
  scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
  # scrape_timeout is set to the global default (10s).

# Alertmanager configuration
alerting:
  alertmanagers:
    - static_configs:
        - targets:
          # - alertmanager:9093

# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
rule_files:
  # - "first_rules.yml"
  # - "second_rules.yml"

# A scrape configuration containing exactly one endpoint to scrape:
# Here it's Prometheus itself.
scrape_configs:
  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
  - job_name: 'prometheus'

    # metrics_path defaults to '/metrics'
    # scheme defaults to 'http'.

    static_configs:
      - targets: ['localhost:9090']

{% for ip in pillar['nodes'] %}
  - job_name: "{{ ip }}"
    static_configs:
      - targets: ["{{ ip }}"]
{% endfor %}

prometheus.service启动文件

[root@master files]# pwd
/srv/salt/prod/modules/prometheus/files
[root@master files]# cat prometheus.service.j2 
[Unit]
Description=The Prometheus Server
After=network.target

[Service]
Restart=on-failure
ExecStart={{ pillar['prometheus_install_dir'] }}/prometheus/prometheus --config.file={{ pillar['prometheus_install_dir'] }}/prometheus/prometheus.yml
ExecStop=/bin/kill -HUP $MAINPID 
RestartSec=15s

[Install]
WantedBy=multi-user.target

执行server.sls

[root@master ~]# salt 'p1' state.sls modules.prometheus.server saltenv=prod
p1:
----------
          ID: copy-grafana-enterprise
    Function: file.managed
        Name: /usr/src/grafana-enterprise-8.2.5-1.x86_64.rpm
      Result: True
     Comment: File /usr/src/grafana-enterprise-8.2.5-1.x86_64.rpm is in the correct state
     Started: 01:04:57.614250
    Duration: 477.94 ms
     Changes:   
----------
          ID: install-grafana
    Function: cmd.run
        Name: cd /usr/src/ && yum -y install grafana-enterprise-8.2.5-1.x86_64.rpm
      Result: True
     Comment: unless condition is true
     Started: 01:04:58.092946
    Duration: 1671.231 ms
     Changes:   
----------
          ID: unzip-prometheus
    Function: archive.extracted
        Name: /usr/local
      Result: True
     Comment: unless condition is true
     Started: 01:04:59.764475
    Duration: 17.066 ms
     Changes:   
----------
          ID: unzip-prometheus
    Function: file.symlink
        Name: /usr/local/prometheus
      Result: True
     Comment: Symlink /usr/local/prometheus is present and owned by root:root
     Started: 01:04:59.781935
    Duration: 7.926 ms
     Changes:   
----------
          ID: copy-files
    Function: file.managed
        Name: /usr/lib/systemd/system/prometheus.service
      Result: True
     Comment: File /usr/lib/systemd/system/prometheus.service is in the correct state
     Started: 01:04:59.789997
    Duration: 13.572 ms
     Changes:   
----------
          ID: copy-files
    Function: file.managed
        Name: /usr/local/prometheus/prometheus.yml
      Result: True
     Comment: File /usr/local/prometheus/prometheus.yml updated
     Started: 01:04:59.803711
    Duration: 20.654 ms
     Changes:   
              ----------
              diff:
                  --- 
                  +++ 
                  @@ -29,9 +29,9 @@
                         - targets: ['localhost:9090']
                   
                   
                  -  - job_name: "192.168.129.135:9100"
                  +  - job_name: "192.168.129.133:9100"
                       static_configs:
                  -      - targets: ["192.168.129.135:9100"]
                  +      - targets: ["192.168.129.133:9100"]
                   
                     - job_name: "192.168.129.136:9100"
                       static_configs:
----------
          ID: start-grafana-enterprise
    Function: service.running
        Name: grafana-server.service
      Result: True
     Comment: The service grafana-server.service is already running
     Started: 01:04:59.824632
    Duration: 41.703 ms
     Changes:   
----------
          ID: start-prometheus
    Function: service.running
        Name: prometheus.service
      Result: True
     Comment: Service prometheus.service is already enabled, and is running
     Started: 01:04:59.867011
    Duration: 97.745 ms
     Changes:   
              ----------
              prometheus.service:
                  True

Summary for p1
------------
Succeeded: 8 (changed=2)
Failed:    0
------------
Total states run:     8
Total run time:   2.348 s

node_exporter状态文件

[root@master prometheus]# pwd
/srv/salt/prod/modules/prometheus
[root@master prometheus]# cat agent.sls 
unzip-node_exporter:
  archive.extracted:
    - name: {{ pillar['node_exporter_install_dir'] }}
    - source: salt://modules/prometheus/files/node_exporter-1.3.0.linux-amd64.tar.gz
    - unless: test -d {{ pillar['node_exporter_install_dir'] }}/node_exporter-1.3.0.linux-amd64

  file.symlink:
    - name: {{ pillar['node_exporter_install_dir'] }}/node_exporter
    - target: {{ pillar['node_exporter_install_dir'] }}/node_exporter-1.3.0.linux-amd64

/usr/lib/systemd/system/node_exporter.service:
  file.managed:
    - source: salt://modules/prometheus/files/node_exporter.service.j2
    - user: root
    - group: root
    - mode: '0644'
    - template: jinja

node_exporter.service:
  service.running:
    - reload: true
    - enable: true
    - watch:
      - file: /usr/lib/systemd/system/node_exporter.service

node_exporter.service启动文件

[root@master prometheus]# cat files/node_exporter.service.j2 
[unit]
Description=The node_exporter Server
After=network.target

[Service]
ExecStart={{ pillar['node_exporter_install_dir'] }}/node_exporter/node_exporter
Restart=on-failure
RestartSec=15s
SyslogIdentifier=node_exporter

[Install]
WantedBy=multi-user.target

执行agent.sls

[root@master ~]# salt 'p2' state.sls modules.prometheus.agent saltenv=prod
p2:
----------
          ID: unzip-node_exporter
    Function: archive.extracted
        Name: /usr/local/
      Result: True
     Comment: unless condition is true
     Started: 00:53:55.563311
    Duration: 1534.873 ms
     Changes:   
----------
          ID: unzip-node_exporter
    Function: file.symlink
        Name: /usr/local//node_exporter
      Result: True
     Comment: Symlink /usr/local//node_exporter is present and owned by root:root
     Started: 00:53:57.098551
    Duration: 5.494 ms
     Changes:   
----------
          ID: /usr/lib/systemd/system/node_exporter.service
    Function: file.managed
      Result: True
     Comment: File /usr/lib/systemd/system/node_exporter.service updated
     Started: 00:53:57.104187
    Duration: 46.98 ms
     Changes:   
              ----------
              diff:
                  New file
              mode:
                  0644
----------
          ID: node_exporter.service
    Function: service.running
      Result: True
     Comment: Service node_exporter.service has been enabled, and is running
     Started: 00:53:57.151820
    Duration: 282.957 ms
     Changes:   
              ----------
              node_exporter.service:
                  True

Summary for p2
------------
Succeeded: 4 (changed=2)
Failed:    0
------------
Total states run:     4
Total run time:   1.870 s

[root@master ~]# salt '*' state.sls modules.prometheus.agent saltenv=prod
[root@master ~]# vim /srv/salt/prod/modules/prometheus/main.sls
prod:
  'p1'
    - modules.prometheus.server
  'p2'
    - modules.prometheus.agent

[root@master ~]# salt '*' state.highstate 

效果图

• 首次登陆时需要ip+端口号(默认为3000)
• 默认账号：admin 密码：admin
  
  再次输入（因为你是首次登陆）