k8s部署grafana beyla
OS:
Static hostname: test
Icon name: computer-vm
Chassis: vm
Machine ID: 22349ac6f9ba406293d0541bcba7c05d
Boot ID: 83bb7e5dbf27453c94ff9f1fe88d5f02
Virtualization: vmware
Operating System: Ubuntu 22.04.4 LTS
Kernel: Linux 5.15.0-105-generic
Architecture: x86-64
Hardware Vendor: VMware, Inc.
Hardware Model: VMware Virtual Platform
kubespray version:
2.25.0
kubernetes version:
1.29.5
部署测试用nginx
cat > nginx.yaml <<EOF
kind: Deployment
apiVersion: apps/v1
metadata:
name: docs
spec:
replicas: 2
selector:
matchLabels:
app: docs
template:
metadata:
labels:
app: docs
spec:
containers:
- name: docs-server
image: httpd:latest
ports:
- containerPort: 80
protocol: TCP
name: http
---
apiVersion: v1
kind: Service
metadata:
name: docs
spec:
selector:
app: docs
ports:
- protocol: TCP
port: 80
---
kind: Deployment
apiVersion: apps/v1
metadata:
name: website
spec:
replicas: 2
selector:
matchLabels:
app: website
template:
metadata:
labels:
app: website
spec:
containers:
- name: website-server
image: dockerhub.timeweb.cloud/httpd:latest
ports:
- containerPort: 80
protocol: TCP
name: http
---
apiVersion: v1
kind: Service
metadata:
name: website
spec:
selector:
app: website
ports:
- protocol: TCP
port: 80
EOF
# 创建
kubectl apply -f nginx.yaml
# 转发端口
kubectl port-forward services/website 8080:80
kubectl port-forward services/docs 8081:80
部署grafana beyla
# 创建命名空间
kubectl create namespace beyla
# 创建serviceaccount
cat > beyla-serviceaccount.yaml <<EOF
apiVersion: v1
kind: ServiceAccount
metadata:
namespace: beyla
name: beyla
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: beyla
rules:
- apiGroups: ["apps"]
resources: ["replicasets"]
verbs: ["list", "watch"]
- apiGroups: [""]
resources: ["pods"]
verbs: ["list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: beyla
subjects:
- kind: ServiceAccount
name: beyla
namespace: beyla
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: beyla
EOF
kubectl apply -f beyla-serviceaccount.yaml
cat > beyla.yaml <<EOF
apiVersion: v1
kind: ConfigMap
metadata:
namespace: beyla
name: beyla-config
data:
beyla-config.yml: |
# this is required to enable kubernetes discovery and metadata
attributes:
kubernetes:
enable: true
# this will provide automatic routes report while minimizing cardinality
routes:
unmatched: heuristic
# let's instrument only the docs server
discovery:
services:
- k8s_deployment_name: "^docs$"
# uncomment the following line to also instrument the website server
# - k8s_deployment_name: "^website$"
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
namespace: beyla
name: beyla
spec:
selector:
matchLabels:
instrumentation: beyla
template:
metadata:
labels:
instrumentation: beyla
spec:
serviceAccountName: beyla
hostPID: true # mandatory!
containers:
- name: beyla
image: dockerhub.timeweb.cloud/grafana/beyla:1.2
imagePullPolicy: IfNotPresent
securityContext:
privileged: true # mandatory!
readOnlyRootFilesystem: true
volumeMounts:
- mountPath: /config
name: beyla-config
- mountPath: /var/run/beyla
name: var-run-beyla
env:
- name: BEYLA_CONFIG_PATH
value: "/config/beyla-config.yml"
- name: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT
value: "http://10.1.1.71:4318/v1/traces"
- name: OTEL_EXPORTER_OTLP_TRACES_PROTOCOL
value: "http/protobuf"
# - name: OTEL_EXPORTER_OTLP_HEADERS
# valueFrom:
# secretKeyRef:
# name: grafana-credentials
# key: otlp-headers
volumes:
- name: beyla-config
configMap:
name: beyla-config
- name: var-run-beyla
emptyDir: {}
EOF
kubectl apply -f beyla.yaml
安装grafana
apt-get install -y adduser libfontconfig1 musl
wget https://dl.grafana.com/oss/release/grafana_10.4.2_amd64.deb
dpkg -i grafana_10.4.2_amd64.deb
systemctl start grafana-server
systemctl enable grafana-server
安装prometheus
wget --no-check-certificate https://github.com/prometheus/prometheus/releases/download/v2.45.4/prometheus-2.45.4.linux-amd64.tar.gz
tar -zxf prometheus-2.45.4.linux-amd64.tar.gz
mkdir -p /etc/prometheus
mkdir -p /export/prometheus/data
cp -r prometheus-2.45.4.linux-amd64/* /etc/prometheus/
mv /etc/prometheus/prometheus /usr/local/bin/
mv /etc/prometheus/promtool /usr/local/bin/
# 配置抓取promttheus
cat <<EOF >/etc/prometheus/prometheus.yml
global:
alerting:
alertmanagers:
- static_configs:
- targets:
# - alertmanager:9093
rule_files:
# - "first_rules.yml"
# - "second_rules.yml"
scrape_configs:
- job_name: "prometheus"
static_configs:
- targets: ["localhost:9090"]
- job_name: "beyla"
static_configs:
- targets: ["localhost:9101", "localhost:9102", "localhost:9103"]
EOF
# 启动
# 使用--web.enable-remote-write-receiver启用远程写入接口来接收tempo的service graph数据,地址为/api/v1/write
screen -dmS prom prometheus --config.file=/etc/prometheus/prometheus.yml --web.enable-remote-write-receiver --storage.tsdb.path=/export/prometheus/data --web.console.libraries=/etc/prometheus/console_libraries --web.console.templates=/etc/prometheus/consoles --storage.tsdb.retention=7d &
安装tempo
安装
curl -Lo tempo_2.4.1_linux_amd64.deb https://github.com/grafana/tempo/releases/download/v2.4.1/tempo_2.4.1_linux_amd64.deb
echo 2fdd167cbb00d732435123a254469ec4cfde3c525a4ec89d235423a5e9abc4b3 \
tempo_2.4.1_linux_amd64.deb | sha256sum -c
dpkg -i tempo_2.4.1_linux_amd64.deb
配置
cat > /etc/tempo/config.yml <<EOF
server:
http_listen_port: 3200
distributor:
receivers:
otlp:
protocols:
http:
grpc:
compactor:
compaction:
block_retention: 48h
metrics_generator:
registry:
external_labels:
source: tempo
cluster: linux-microservices
storage:
path: /tmp/tempo/generator/wal
remote_write:
- url: http://localhost:9090/api/v1/write
send_exemplars: true
storage:
# trace:
# backend: s3
# s3:
# endpoint: s3.us-east-1.amazonaws.com
# bucket: grafana-traces-data
# forcepathstyle: true
# # set to false if endpoint is https
# insecure: true
# access_key: # TODO - Add S3 access key
# secret_key: # TODO - Add S3 secret key
trace:
backend: local
wal:
path: /tmp/tempo/wal
local:
path: /tmp/tempo/blocks
overrides:
defaults:
metrics_generator:
processors: [service-graphs, span-metrics]
EOF
启动
systemctl start tempo.service
systemctl enable tempo.service
systemctl is-active tempo
生成trace数据
访问生成trace信息
curl http://localhost:8080
curl http://localhost:8080/foo
curl http://localhost:8081
curl http://localhost:8081/foo
tail pod日志
kubectl logs -f beyla-spb9s -n beyla
grafana面板配置
添加tempo源并启用node graph
启用service graph
prometheus需要使用–web.enable-remote-write-receiver启用远程写入接口来接收tempo的service graph数据,地址为/api/v1/write
prometheus http API
在explorer中搜索service graph
设置hosts
# vim /etc/hosts
127.0.0.1 prometheus