使用prometheus监控k8s集群+grafana出图

prometheus官方：https://prometheus.io/

grafana官方： https://grafana.com/

1.部署prometheus

我这里是采用的二进制安装，写了个脚本；

包可以提前下好上传到主机，内网环境下wget不下来会安装失败；

sh install_prometheus

vim install_prometheus

#!/bin/bash
#
#********************************************************************
#Author:            jiangxiamiao
#FileName:          install_prometheus.sh

#********************************************************************

PROMETHEUS_VERSION=2.34.0
#PROMETHEUS_VERSION=2.17.1
PROMETHEUS_FILE="prometheus-${PROMETHEUS_VERSION}.linux-amd64.tar.gz"
INSTALL_DIR=/usr/local

HOST=`hostname -I|awk '{print $1}'`


. /etc/os-release

msg_error() {
  echo -e "\033[1;31m$1\033[0m"
}

msg_info() {
  echo -e "\033[1;32m$1\033[0m"
}

msg_warn() {
  echo -e "\033[1;33m$1\033[0m"
}


color () {
    RES_COL=60
    MOVE_TO_COL="echo -en \\033[${RES_COL}G"
    SETCOLOR_SUCCESS="echo -en \\033[1;32m"
    SETCOLOR_FAILURE="echo -en \\033[1;31m"
    SETCOLOR_WARNING="echo -en \\033[1;33m"
    SETCOLOR_NORMAL="echo -en \E[0m"
    echo -n "$1" && $MOVE_TO_COL
    echo -n "["
    if [ $2 = "success" -o $2 = "0" ] ;then
        ${SETCOLOR_SUCCESS}
        echo -n $"  OK  "    
    elif [ $2 = "failure" -o $2 = "1"  ] ;then 
        ${SETCOLOR_FAILURE}
        echo -n $"FAILED"
    else
        ${SETCOLOR_WARNING}
        echo -n $"WARNING"
    fi
    ${SETCOLOR_NORMAL}
    echo -n "]"
    echo 
}


install_prometheus () {
    if [ ! -f  ${PROMETHEUS_FILE} ] ;then
        wget https://github.com/prometheus/prometheus/releases/download/v${PROMETHEUS_VERSION}/${PROMETHEUS_FILE} || \
            { color "下载失败!" 1 ; exit ; }
    fi
    [ -d $INSTALL_DIR ] || mkdir -p $INSTALL_DIR
    tar xf ${PROMETHEUS_FILE} -C $INSTALL_DIR
    cd $INSTALL_DIR &&  ln -s prometheus-${PROMETHEUS_VERSION}.linux-amd64 prometheus
    mkdir -p $INSTALL_DIR/prometheus/{bin,conf,data}
    cd $INSTALL_DIR/prometheus && { mv prometheus promtool bin/ ; mv prometheus.yml conf/; }
    useradd -r -s /sbin/nologin prometheus
    chown -R prometheus.prometheus ${INSTALL_DIR}/prometheus/
    
    cat >  /etc/profile.d/prometheus.sh <<EOF
export PROMETHEUS_HOME=${INSTALL_DIR}/prometheus
export PATH=\${PROMETHEUS_HOME}/bin:\$PATH
EOF

}


prometheus_service () {
    cat > /lib/systemd/system/prometheus.service <<EOF
[Unit]
Description=Prometheus Server
Documentation=https://prometheus.io/docs/introduction/overview/
After=network.target

[Service]
Restart=on-failure
User=prometheus
WorkingDirectory=${INSTALL_DIR}/prometheus
ExecStart=${INSTALL_DIR}/prometheus/bin/prometheus --config.file=${INSTALL_DIR}/prometheus/conf/prometheus.yml
ExecReload=/bin/kill -HUP $MAINPID
LimitNOFILE=65535

[Install]
WantedBy=multi-user.target
EOF
    systemctl daemon-reload
    systemctl enable --now prometheus.service
}


start_prometheus() { 
    systemctl is-active prometheus
    if [ $?  -eq 0 ];then  
        echo 
        color "Prometheus 安装完成!" 0
        echo "-------------------------------------------------------------------"
        echo -e "访问链接: \c"
        msg_info "http://$HOST:9090/" 
    else
        color "Prometheus 安装失败!" 1
        exit
    fi 
}

install_prometheus

prometheus_service

start_prometheus

2.部署Node_exporter

部署node_exporter这边我也是采用的二进制，写了个脚本；

在每台k8s集群的node节点上上传脚本，并执行脚本；

端口是9100；

sh install_node_exporter

vim install_node_exporter

#!/bin/bash
#
#********************************************************************
#Author:            jiangxiamiao
#FileName:          install_node_exporter.sh

#********************************************************************

NODE_EXPORTER_VERSION=1.3.1
NODE_EXPORTER_FILE="node_exporter-${NODE_EXPORTER_VERSION}.linux-amd64.tar.gz"
INSTALL_DIR=/usr/local

HOST=`hostname -I|awk '{print $1}'`


. /etc/os-release

msg_error() {
  echo -e "\033[1;31m$1\033[0m"
}

msg_info() {
  echo -e "\033[1;32m$1\033[0m"
}

msg_warn() {
  echo -e "\033[1;33m$1\033[0m"
}


color () {
    RES_COL=60
    MOVE_TO_COL="echo -en \\033[${RES_COL}G"
    SETCOLOR_SUCCESS="echo -en \\033[1;32m"
    SETCOLOR_FAILURE="echo -en \\033[1;31m"
    SETCOLOR_WARNING="echo -en \\033[1;33m"
    SETCOLOR_NORMAL="echo -en \E[0m"
    echo -n "$1" && $MOVE_TO_COL
    echo -n "["
    if [ $2 = "success" -o $2 = "0" ] ;then
        ${SETCOLOR_SUCCESS}
        echo -n $"  OK  "    
    elif [ $2 = "failure" -o $2 = "1"  ] ;then 
        ${SETCOLOR_FAILURE}
        echo -n $"FAILED"
    else
        ${SETCOLOR_WARNING}
        echo -n $"WARNING"
    fi
    ${SETCOLOR_NORMAL}
    echo -n "]"
    echo 
}


install_node_exporter () {
    if [ ! -f  ${NODE_EXPORTER_FILE} ] ;then
        wget https://github.com/prometheus/node_exporter/releases/download/v${NODE_EXPORTER_VERSION}/${NODE_EXPORTER_FILE} || \
            { color "下载失败!" 1 ; exit ; }
    fi
    [ -d $INSTALL_DIR ] || mkdir -p $INSTALL_DIR
    tar xf ${NODE_EXPORTER_FILE} -C $INSTALL_DIR
    cd $INSTALL_DIR &&  ln -s node_exporter-${NODE_EXPORTER_VERSION}.linux-amd64 node_exporter
    mkdir -p $INSTALL_DIR/node_exporter/bin
    cd $INSTALL_DIR/node_exporter &&  mv node_exporter bin/ 
      
    cat >  /etc/profile.d/node_exporter.sh <<EOF
export NODE_EXPORTER_HOME=${INSTALL_DIR}/node_exporter
export PATH=\${NODE_EXPORTER_HOME}/bin:\$PATH
EOF

}


node_exporter_service () {
    cat > /lib/systemd/system/node_exporter.service <<EOF
[Unit]
Description=Prometheus Node Exporter
After=network.target

[Service]
Type=simple
ExecStart=$INSTALL_DIR/node_exporter/bin/node_exporter
Restart=on-failure

[Install]
WantedBy=multi-user.target
EOF
    systemctl daemon-reload
    systemctl enable --now node_exporter.service
}


start_node_exporter() { 
    systemctl is-active node_exporter.service
    if [ $?  -eq 0 ];then  
        echo 
        color "node_exporter 安装完成!" 0
        echo "-------------------------------------------------------------------"
        echo -e "访问链接: \c"
        msg_info "http://$HOST:9100/metrics" 
    else
        color "node_exporter 安装失败!" 1
        exit
    fi 
}

install_node_exporter

node_exporter_service

start_node_exporter

3.部署Advisor

这里我采用的是docker的方式部署的；

我映射的端口是8080；

在每台k8s集群node上执行此操作；

docker run --volume=/:/rootfs:ro \
--volume=/var/run:/var/run:rw \
--volume=/sys:/sys:ro \
--volume=/var/lib/docker/:/var/lib/docker:ro \
--volume=/dev/disk/:/dev/disk:ro \
--publish=8080:8080 \
--detach=true \
--name=cadvisor \
docker.io/google/cadvisor:latest

4.prometheus采集数据

修改配置文件；在末尾添加如下信息；

vim /usr/local/prometheus/conf/prometheus.yml 

  - job_name: "node_exporter"
    static_configs:
      - targets: ['10.0.0.14:9100','10.0.0.15:9100','10.0.0.16:9100']
  - job_name: "cadvisor"
    static_configs:
      - targets: ['10.0.0.14:8080','10.0.0.15:8080','10.0.0.16:8080']

重启；

systemctl restart prometheus

5.grafana展示出图

部署grafana；这边我采用的yum安装；默认用户名admin，密码admin；

yum install grafana-enterprise.x86_64 -y

systemctl enable --now grafana-server.service 


浏览器访问：http://ip:3000

选择数据源；

 

 

 

访问grafana的官网，选择合适模板的id；复制id； 

---

点+号选择import，导入刚刚复制的id；点load；