监控Tendermint多节点资源使用情况

通过Docker Engine的API获取Docker各个容器的资源使用情况

1. 配置docker-compose.yml文件，增加/var/run/docker.sock:/var/run/docker.sock，将 Docker 引擎的套接字挂载到容器内部

services:
  node0:
    container_name: node0
    image: "tendermint/localnode1"
    ports:
      - "26656-26657:26656-26657"
    environment:
      - ID=0
      - LOG=${LOG:-tendermint.log}
    volumes:
      - ./build:/tendermint:Z
      - /etc/localtime:/etc/localtime
      - /var/run/docker.sock:/var/run/docker.sock
    networks:
      localnet:
        ipv4_address: 192.168.10.2

2.修改Tendermint源码，在p2p/metrics.go增加CpuUsage Metrics

type Metrics struct {
	// CPU使用情况
	CpuUsage metrics.Gauge
}

func PrometheusMetrics(namespace string, labelsAndValues ...string) *Metrics {
	labels := []string{}
	for i := 0; i < len(labelsAndValues); i += 2 {
		labels = append(labels, labelsAndValues[i])
	}
	return &Metrics{
		CpuUsage: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
			Namespace: namespace,
			Subsystem: MetricsSubsystem,
			Name:      "cpuUsage",
			Help:      "Usage of CPU.",
		}, append(labels, "container_id")).With(labelsAndValues...),

3.在p2p/peer.go文件中的onReceive中增加获取docker容器使用情况的代码（cpuUsage、memoryUsage、NetworkIO、diskRead/Write），每次接受消息时获取一次docker容器的资源使用情况（以cpuUsage为例）
（1）通过Docker 引擎的套接字获取容器的资源使用情况

//参数strem=false 代表静态获取一次容器的使用数据
out, err := exec.Command("curl", "--unix-socket", "/var/run/docker.sock", "http:/v1.42/containers/node0/stats?stream=false").Output()

（2） 将获取到的数据解析为json格式数据

stats := make(map[string]interface{})
	err = json.Unmarshal(out, &stats)

（3）根据Docker 引擎API计算指标的公式解析所需的数据

cpuStats := stats["cpu_stats"].(map[string]interface{})
	cpuUsage := cpuStats["cpu_usage"].(map[string]interface{})
	totalUsage := cpuUsage["total_usage"].(float64)
	systemUsage := cpuStats["system_cpu_usage"].(float64)

	preCpuStats := stats["precpu_stats"].(map[string]interface{})
	preCpuUsage := preCpuStats["cpu_usage"].(map[string]interface{})
	PreTotalUsage := preCpuUsage["total_usage"].(float64)
	preSystemUsage := preCpuStats["system_cpu_usage"].(float64)
    numberCpus := cpuStats["online_cpus"].(float64)

（4）根据Docker 引擎API计算指标的公式计算指标值

cpuDelta := totalUsage - PreTotalUsage
	systemCpuDelta := systemUsage - preSystemUsage
	cpuPercent := 0.0
	//numCores := float64(len(cpuUsage["percpu_usage"].([]interface{})))
	if numberCpus > 0 {	
		cpuPercent = (cpuDelta / systemCpuDelta) * numberCpus * 100.0
	}

示例代码如下

out, err := exec.Command("curl", "--unix-socket", "/var/run/docker.sock", "http:/v1.42/containers/node0/stats?stream=false").Output()
	if err != nil {
		panic(err)
	}
	stats := make(map[string]interface{})
	err = json.Unmarshal(out, &stats)
	if err != nil {
		panic(err)
	}

	cpuStats := stats["cpu_stats"].(map[string]interface{})
	cpuUsage := cpuStats["cpu_usage"].(map[string]interface{})
	totalUsage := cpuUsage["total_usage"].(float64)
	systemUsage := cpuStats["system_cpu_usage"].(float64)

	preCpuStats := stats["precpu_stats"].(map[string]interface{})
	preCpuUsage := preCpuStats["cpu_usage"].(map[string]interface{})
	PreTotalUsage := preCpuUsage["total_usage"].(float64)
	preSystemUsage := preCpuStats["system_cpu_usage"].(float64)
    numberCpus := cpuStats["online_cpus"].(float64)
    
	cpuDelta := totalUsage - PreTotalUsage
	systemCpuDelta := systemUsage - preSystemUsage
	
	cpuPercent := 0.0
	//numCores := float64(len(cpuUsage["percpu_usage"].([]interface{})))
	if numberCpus > 0 {
		//cpuPercent = (totalUsage / systemUsage) * numCores * 100.0
		cpuPercent = (cpuDelta / systemCpuDelta) * numberCpus * 100.0
	}

	fmt.Printf("CPU Usage: %.2f%%\n", cpuPercent)
   p.metrics.CpuUsage.With("container_id", "node0").Set(cpuPercent)

4.编译源码 开启网络测试

cd ~/workspace/tendermint
make build
make localnet-start

在浏览器输入

http://localhost:9090/

示例如下：