k8s部署grafana beyla实现app应用服务依赖图可观测

k8s部署grafana beyla

OS:
Static hostname: test
Icon name: computer-vm
Chassis: vm
Machine ID: 22349ac6f9ba406293d0541bcba7c05d
Boot ID: 83bb7e5dbf27453c94ff9f1fe88d5f02
Virtualization: vmware
Operating System: Ubuntu 22.04.4 LTS
Kernel: Linux 5.15.0-105-generic
Architecture: x86-64
Hardware Vendor: VMware, Inc.
Hardware Model: VMware Virtual Platform

kubespray version:
2.25.0

kubernetes version:
1.29.5

部署测试用nginx

cat > nginx.yaml <<EOF
kind: Deployment
apiVersion: apps/v1
metadata:
  name: docs
spec:
  replicas: 2
  selector:
    matchLabels:
      app: docs
  template:
    metadata:
      labels:
        app: docs
    spec:
      containers:
        - name: docs-server
          image: httpd:latest
          ports:
            - containerPort: 80
              protocol: TCP
              name: http
---
apiVersion: v1
kind: Service
metadata:
  name: docs
spec:
  selector:
    app: docs
  ports:
    - protocol: TCP
      port: 80
---
kind: Deployment
apiVersion: apps/v1
metadata:
  name: website
spec:
  replicas: 2
  selector:
    matchLabels:
      app: website
  template:
    metadata:
      labels:
        app: website
    spec:
      containers:
        - name: website-server
          image: dockerhub.timeweb.cloud/httpd:latest
          ports:
            - containerPort: 80
              protocol: TCP
              name: http
---
apiVersion: v1
kind: Service
metadata:
  name: website
spec:
  selector:
    app: website
  ports:
    - protocol: TCP
      port: 80
EOF
# 创建
kubectl apply -f nginx.yaml
# 转发端口
kubectl port-forward services/website 8080:80
kubectl port-forward services/docs 8081:80

部署grafana beyla

# 创建命名空间
kubectl create namespace beyla
# 创建serviceaccount
cat > beyla-serviceaccount.yaml <<EOF
apiVersion: v1
kind: ServiceAccount
metadata:
  namespace: beyla
  name: beyla
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: beyla
rules:
  - apiGroups: ["apps"]
    resources: ["replicasets"]
    verbs: ["list", "watch"]
  - apiGroups: [""]
    resources: ["pods"]
    verbs: ["list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: beyla
subjects:
  - kind: ServiceAccount
    name: beyla
    namespace: beyla
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: beyla
EOF

kubectl apply -f beyla-serviceaccount.yaml

cat > beyla.yaml <<EOF
apiVersion: v1
kind: ConfigMap
metadata:
  namespace: beyla
  name: beyla-config
data:
  beyla-config.yml: |
    # this is required to enable kubernetes discovery and metadata
    attributes:
      kubernetes:
        enable: true
    # this will provide automatic routes report while minimizing cardinality
    routes:
      unmatched: heuristic
    # let's instrument only the docs server
    discovery:
      services:
        - k8s_deployment_name: "^docs$"
        # uncomment the following line to also instrument the website server
        # - k8s_deployment_name: "^website$"
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  namespace: beyla
  name: beyla
spec:
  selector:
    matchLabels:
      instrumentation: beyla
  template:
    metadata:
      labels:
        instrumentation: beyla
    spec:
      serviceAccountName: beyla
      hostPID: true # mandatory!
      containers:
        - name: beyla
          image: dockerhub.timeweb.cloud/grafana/beyla:1.2
          imagePullPolicy: IfNotPresent
          securityContext:
            privileged: true # mandatory!
            readOnlyRootFilesystem: true
          volumeMounts:
            - mountPath: /config
              name: beyla-config
            - mountPath: /var/run/beyla
              name: var-run-beyla
          env:
            - name: BEYLA_CONFIG_PATH
              value: "/config/beyla-config.yml"
            - name: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT
              value: "http://10.1.1.71:4318/v1/traces"
            - name: OTEL_EXPORTER_OTLP_TRACES_PROTOCOL
              value: "http/protobuf"
#            - name: OTEL_EXPORTER_OTLP_HEADERS
#              valueFrom:
#                secretKeyRef:
#                  name: grafana-credentials
#                  key: otlp-headers
      volumes:
        - name: beyla-config
          configMap:
            name: beyla-config
        - name: var-run-beyla
          emptyDir: {}
EOF

kubectl apply -f beyla.yaml

安装grafana

apt-get install -y adduser libfontconfig1 musl
wget https://dl.grafana.com/oss/release/grafana_10.4.2_amd64.deb
dpkg -i grafana_10.4.2_amd64.deb
systemctl start grafana-server
systemctl enable grafana-server

安装prometheus

wget --no-check-certificate https://github.com/prometheus/prometheus/releases/download/v2.45.4/prometheus-2.45.4.linux-amd64.tar.gz
tar -zxf prometheus-2.45.4.linux-amd64.tar.gz
mkdir -p /etc/prometheus
mkdir -p /export/prometheus/data
cp -r prometheus-2.45.4.linux-amd64/* /etc/prometheus/
mv /etc/prometheus/prometheus /usr/local/bin/
mv /etc/prometheus/promtool /usr/local/bin/

# 配置抓取promttheus
cat <<EOF >/etc/prometheus/prometheus.yml
global:
alerting:
  alertmanagers:
    - static_configs:
        - targets:
          # - alertmanager:9093
rule_files:
  # - "first_rules.yml"
  # - "second_rules.yml"
scrape_configs:
  - job_name: "prometheus"
    static_configs:
      - targets: ["localhost:9090"]
  - job_name: "beyla"
    static_configs:
      - targets: ["localhost:9101", "localhost:9102", "localhost:9103"]
EOF
# 启动
# 使用--web.enable-remote-write-receiver启用远程写入接口来接收tempo的service graph数据，地址为/api/v1/write
screen -dmS prom prometheus --config.file=/etc/prometheus/prometheus.yml --web.enable-remote-write-receiver --storage.tsdb.path=/export/prometheus/data --web.console.libraries=/etc/prometheus/console_libraries --web.console.templates=/etc/prometheus/consoles --storage.tsdb.retention=7d &

安装tempo

tempo部署
tempo配置

安装

curl -Lo tempo_2.4.1_linux_amd64.deb https://github.com/grafana/tempo/releases/download/v2.4.1/tempo_2.4.1_linux_amd64.deb
echo 2fdd167cbb00d732435123a254469ec4cfde3c525a4ec89d235423a5e9abc4b3 \
  tempo_2.4.1_linux_amd64.deb | sha256sum -c
dpkg -i tempo_2.4.1_linux_amd64.deb

配置

cat > /etc/tempo/config.yml <<EOF
server:
  http_listen_port: 3200

distributor:
  receivers:
      otlp:
        protocols:
          http:
          grpc:

compactor:
  compaction:
    block_retention: 48h

metrics_generator:
  registry:
    external_labels:
      source: tempo
      cluster: linux-microservices
  storage:
    path: /tmp/tempo/generator/wal
    remote_write:
    - url: http://localhost:9090/api/v1/write
      send_exemplars: true

storage:
#   trace:
#     backend: s3
#     s3:
#       endpoint: s3.us-east-1.amazonaws.com
#       bucket: grafana-traces-data
#       forcepathstyle: true
#       # set to false if endpoint is https
#       insecure: true
#       access_key: # TODO - Add S3 access key
#       secret_key: # TODO - Add S3 secret key

  trace:
    backend: local
    wal:
      path: /tmp/tempo/wal
    local:
      path: /tmp/tempo/blocks
overrides:
  defaults:
    metrics_generator:
      processors: [service-graphs, span-metrics]
EOF

启动

systemctl start tempo.service
systemctl enable tempo.service
systemctl is-active tempo

生成trace数据

访问生成trace信息

curl http://localhost:8080
curl http://localhost:8080/foo
curl http://localhost:8081
curl http://localhost:8081/foo

tail pod日志

kubectl logs -f beyla-spb9s -n beyla

grafana面板配置

添加tempo源并启用node graph

启用service graph

prometheus需要使用–web.enable-remote-write-receiver启用远程写入接口来接收tempo的service graph数据，地址为/api/v1/write
prometheus http API

在explorer中搜索service graph

设置hosts

# vim /etc/hosts
127.0.0.1 prometheus