Prometheus

Prometheus

功能

• 在业务层用作埋点系统
  Prometheus支持多种语言（Go，java，python，ruby官方提供客户端，其他语言有第三方开源客户端）。我们可以通过客户端方面的对核心业务进行埋点。如下单流程、添加购物车流程。

• 在应用层用作应用监控系统
  一些主流应用可以通过官方或第三方的导出器，来对这些应用做核心指标的收集。如redis,mysql。

• 在系统层用作系统监控
  除了常用软件， prometheus也有相关系统层和网络层exporter,用以监控服务器或网络。

• 集成其他的监控
  prometheus还可以通过各种exporte，集成其他的监控系统，收集监控数据，如AWS CloudWatch,JMX，Pingdom等等

架构

部署

由于Prometheus 是用golang开发的，所以首先安装一个go环境，Go语言是跨平台，支持Windows、Linux、Mac OS X等系统，还提供有源码，可编译安装

go环境 官网.
Prometheus 官网.

安装go

[root@node3 src]# ls
debug  go1.17.3.linux-amd64.tar.gz  kernels  prometheus-2.32.0-beta.0.linux-amd64.tar.gz
[root@node3 src]# tar xf go1.17.3.linux-amd64.tar.gz -C /usr/local/

[root@node3 ~]# vim /etc/profile
export PATH=$PATH:/usr/local/go/bin

[root@node3 ~]# source  /etc/profile

[root@node3 ~]# go version
go version go1.17.3 linux/amd64

安装prometheus

[root@node3 src]# tar xf prometheus-2.32.0-beta.0.linux-amd64.tar.gz -C /usr/local/

[root@node3 src]# cd /usr/local/
[root@node3 local]# ls
bin  etc  games  go  include  lib  lib64  libexec  prometheus-2.32.0-beta.0.linux-amd64  sbin  share  src
[root@node3 local]# mv prometheus-2.32.0-beta.0.linux-amd64 prometheus

[root@node3 local]# cd prometheus/
[root@node3 prometheus]# ls
console_libraries  consoles  LICENSE  NOTICE  prometheus  prometheus.yml  promtool

#运行后不要退出
[root@node3 prometheus]# ./prometheus 
ts=2021-11-25T11:16:48.349Z caller=main.go:478 level=info msg="No time or size retention was set so using the default time retention" duration=15d
ts=2021-11-25T11:16:48.349Z caller=main.go:515 level=info msg="Starting Prometheus" version="(version=2.32.0-beta.0, branch=HEAD, revision=c32725ba7873dbaa39c223410043430ffa5a26c0)"
ts=2021-11-25T11:16:48.349Z caller=main.go:520 level=info build_context="(go=go1.17.3, user=root@da630543d231, date=20211116-11:23:14)"
ts=2021-11-25T11:16:48.349Z caller=main.go:521 level=info host_details="(Linux 4.18.0-193.el8.x86_64 #1 SMP Fri Mar 27 14:35:58 UTC 2020 x86_64 node3 (none))"
ts=2021-11-25T11:16:48.349Z caller=main.go:522 level=info fd_limits="(soft=1024, hard=262144)"
ts=2021-11-25T11:16:48.349Z caller=main.go:523 level=info vm_limits="(soft=unlimited, hard=unlimited)"
ts=2021-11-25T11:16:48.351Z caller=web.go:546 level=info component=web msg="Start listening for connections" address=0.0.0.0:9090
ts=2021-11-25T11:16:48.351Z caller=main.go:924 level=info msg="Starting TSDB ..."
ts=2021-11-25T11:16:48.353Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false
ts=2021-11-25T11:16:48.391Z caller=head.go:479 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any"
ts=2021-11-25T11:16:48.391Z caller=head.go:513 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.065µs
ts=2021-11-25T11:16:48.391Z caller=head.go:519 level=info component=tsdb msg="Replaying WAL, this may take a while"
ts=2021-11-25T11:16:48.391Z caller=head.go:590 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0
ts=2021-11-25T11:16:48.391Z caller=head.go:596 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=32.857µs wal_replay_duration=327.908µs total_replay_duration=391.694µs
ts=2021-11-25T11:16:48.392Z caller=main.go:945 level=info fs_type=XFS_SUPER_MAGIC
ts=2021-11-25T11:16:48.392Z caller=main.go:948 level=info msg="TSDB started"
ts=2021-11-25T11:16:48.392Z caller=main.go:1129 level=info msg="Loading configuration file" filename=prometheus.yml
ts=2021-11-25T11:16:48.511Z caller=main.go:1166 level=info msg="Completed loading of configuration file" filename=prometheus.yml totalDuration=119.571209ms db_storage=680ns remote_storage=1.534µs web_handler=519ns query_engine=837ns scrape=119.307698ms scrape_sd=21.542µs notify=23.976µs notify_sd=9.204µs rules=1.09µs
ts=2021-11-25T11:16:48.512Z caller=main.go:897 level=info msg="Server is ready to receive web requests."


[root@node3 ~]# ss -antl
State          Recv-Q         Send-Q                   Local Address:Port                   Peer Address:Port         
LISTEN         0              128                            0.0.0.0:22                          0.0.0.0:*            
LISTEN         0              128                                  *:9090                              *:*            
LISTEN         0              128                               [::]:22                             [::]:*   

编写service文件

[root@node3 ~]# vim /usr/lib/systemd/system/prometheus.service
[Unit]
Description=prometheus
After=network.target

[Service]
WorkingDirectory=/usr/local/prometheus
ExecStart=/usr/local/prometheus/prometheus
[Install]
WantedBy=multi-user.target

[root@node3 ~]# systemctl enable --now prometheus.service 
Created symlink /etc/systemd/system/multi-user.target.wants/prometheus.service → /usr/lib/systemd/system/prometheus.service.
[root@node3 ~]# systemctl status prometheus.service 
● prometheus.service - prometheus
   Loaded: loaded (/usr/lib/systemd/system/prometheus.service; enabled; vendor preset: disabled)
   Active: active (running) since Thu 2021-11-25 19:30:35 CST; 7s ago

监控httpd服务

在被控节点安装httpd服务和apache_export

[root@node2 ~]# yum -y install httpd

[root@node2 ~]# systemctl enable --now httpd

[root@node2 src]# wget https://github.com/Lusitaniae/apache_exporter/releases/download/v0.5.0/apache_exporter-0.5.0.linux-amd64.tar.gz

[root@node2 src]# ls
apache_exporter-0.5.0.linux-amd64.tar.gz

[root@node2 src]# tar xf apache_exporter-0.5.0.linux-amd64.tar.gz -C /usr/local/apache_export

[root@node2 ~]# cd /usr/local/
[root@node2 local]# cd apache_export/
[root@node2 apache_export]# ./apache_exporter 
INFO[0000] Starting apache_exporter (version=0.5.0, branch=HEAD, revision=f6a5b4814ea795ee9eac745c55649cce9e5117a9)  source="apache_exporter.go:305"
INFO[0000] Build context (go=go1.9.2, user=root@0fdc4d8924f5, date=20171113-21:19:13)  source="apache_exporter.go:306"
INFO[0000] Starting Server: :9117                        source="apache_exporter.go:307"

[root@node2 apache_export]# ss -antl
State               Recv-Q              Send-Q                            Local Address:Port                             Peer Address:Port              
LISTEN              0                   128                                     0.0.0.0:22                                    0.0.0.0:*                 
LISTEN              0                   128                                     0.0.0.0:6379                                  0.0.0.0:*                 
LISTEN              0                   128                                           *:80                                          *:*                 
LISTEN              0                   128                                        [::]:22                                       [::]:*                 
LISTEN              0                   128                                           *:9117                                        *:* 

修改prometheus配置

[root@node3 ~]# systemctl restart prometheus.service
  - job_name: "prometheus"

    # metrics_path defaults to '/metrics'
    # scheme defaults to 'http'.

    static_configs:
      - targets: ["192.168.8.132:9117"]

[root@node3 ~]# systemctl restart prometheus.service