grafana部署时集成自定义datasource和dashboard页面
做交付项目时,产品有一个grafana,grafana里有自己定义的datasource和dashboard监控模版。如何在grafana部署时就自动带上这些dashboard模版,不需要部署完成之后手动导入dashboard模版。
1. grafana的configmap
在grafana的configmap里面增加datasources.yaml和dashboardproviders.yaml。
datasources.yaml里对应了默认增加的数据源的配置,我自定义了两个数据源,分别是prometheus和Loki类型的。
dashboardproviders.yaml里自定义了两个dashboard模版文件,模版文件放在对应的path目录下。
---
# Source: grafana/templates/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: grafana
labels:
helm.sh/chart: grafana-6.18.2
app.kubernetes.io/name: grafana
app.kubernetes.io/instance: grafana
app.kubernetes.io/version: "8.3.1"
app.kubernetes.io/managed-by: Helm
data:
grafana.ini: |
[analytics]
check_for_updates = true
[grafana_net]
url = https://grafana.net
[log]
mode = console
[server]
root_url=/0/grafana
serve_from_sub_path = true
[paths]
data = /var/lib/grafana/
logs = /var/log/grafana
plugins = /var/lib/grafana/plugins
provisioning = /etc/grafana/provisioning ## 指定grafana 自定义数据源和dashboard模版目录路径
datasources.yaml: |
apiVersion: 1
datasources:
- access: proxy
editable: true
isDefault: true
uid: PBFA97CFB590B2093
jsonData:
timeInterval: 5s
name: Prometheus
orgId: 1
type: prometheus
url: http://prometheus:9090
- access: proxy
editable: true
uid: P8E80F9AEF21F6940
isDefault: false
jsonData:
timeInterval: 5s
name: Loki
orgId: 1
type: loki
url: http://loki:3100
dashboardproviders.yaml: |
apiVersion: 1
providers:
- disableDeletion: false
folder: ''
name: test-model
options:
path: /var/lib/grafana/dashboards/test-model
orgId: 1
type: file
- disableDeletion: false
folder: ''
name: test-logs
options:
path: /var/lib/grafana/dashboards/test-logs
orgId: 1
type: file
2. 增加dashboard configmap
将自定义的dashboard导出为json。
将json文件内容拷贝到configmap里对应的key。
请注意:
导出的dashboard模版文件中引用的数据源的uid是之前手动添加时生成的,所以找出导出dashboard模版里对应datasource的uid,在第一个configmap里面使用 uid: 配置。使其grafana部署后集成的数据源和模版匹配。
---
apiVersion: v1
data:
model.json: |
{"annotations":{"list":[{"builtIn":1,"datasource":"-- Grafana --","enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","target":{"limit":100,"matchAny":false,"tags":[],"type":"dashboard"},"type":"dashboard"}]},"editable":true,"fiscalYearStartMonth":0,"graphTooltip":0,"id":1,"iteration":1661305270887,"links":[],"liveNow":false,"panels":[{"fieldConfig":{"defaults":{"color":{"mode":"palette-classic","seriesBy":"last"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":23,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineStyle":{"fill":"solid"},"lineWidth":1,"pointSize":1,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":9,"w":12,"x":0,"y":0},"id":2,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom"},"tooltip":{"mode":"single"}},"targets":[{"datasource":{"type":"prometheus","uid":"PBFA97CFB590B2093"},"exemplar":true,"expr":"sum(rate(container_cpu_usage_seconds_total{container=~\"$container\",container!=\"\"}[5m])) by(container)","format":"time_series","instant":false,"interval":"","intervalFactor":1,"legendFormat":"{{`{{pod}}`}}","refId":"A"}],"title":"CPU Usage","type":"timeseries"},{"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":23,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":1,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":9,"w":12,"x":12,"y":0},"id":4,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom"},"tooltip":{"mode":"single"}},"targets":[{"datasource":{"type":"prometheus","uid":"PBFA97CFB590B2093"},"exemplar":true,"expr":"sum(container_memory_working_set_bytes{container=~\"$container\", container!=\"\", image!=\"\"}) by (container)","interval":"","intervalFactor":1,"legendFormat":"{{`{{container}}`}}","refId":"A"}],"title":"Memory Usage","type":"timeseries"},{"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":23,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":1,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":9},"id":6,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom"},"tooltip":{"mode":"single"}},"targets":[{"datasource":{"type":"prometheus","uid":"PBFA97CFB590B2093"},"exemplar":true,"expr":"sum(irate(container_network_receive_bytes_total{pod=~\"${container}-.*\"}[$__rate_interval])) by (pod)","interval":"","legendFormat":"{{`{{pod}}`}}","refId":"A"}],"title":"Receive Bandwidth","type":"timeseries"},{"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":23,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":1,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":9},"id":8,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom"},"tooltip":{"mode":"single"}},"targets":[{"datasource":{"type":"prometheus","uid":"PBFA97CFB590B2093"},"exemplar":true,"expr":"sum(irate(container_network_transmit_bytes_total{pod=~\"${container}-.*\"}[$__rate_interval])) by (pod)","interval":"","legendFormat":"{{`{{pod}}`}}","refId":"A"}],"title":"Transmit Bandwidth","type":"timeseries"}],"refresh":"10s","schemaVersion":33,"style":"dark","tags":[],"templating":{"list":[{"current":{"selected":true,"text":["model-63057f4b4a9c64ffa3ceb1a3"],"value":["model-63057f4b4a9c64ffa3ceb1a3"]},"definition":"label_values(container_cpu_usage_seconds_total{namespace=\"{{ .Release.Namespace }}\"},container)","hide":0,"includeAll":false,"multi":true,"name":"container","options":[],"query":{"query":"label_values(container_cpu_usage_seconds_total{namespace=\"{{ .Release.Namespace }}\"},container)","refId":"StandardVariableQuery"},"refresh":1,"regex":"/.*-model.*/","skipUrlSync":false,"sort":1,"type":"query"}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"hidden":false},"timezone":"","title":"model","uid":"UDGZ0SZ4k","version":5,"weekStart":""}
logs.json: |
{"annotations":{"list":[{"$$hashKey":"object:75","builtIn":1,"datasource":"-- Grafana --","enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","target":{"limit":100,"matchAny":false,"tags":[],"type":"dashboard"},"type":"dashboard"}]},"description":"Log Viewer Dashboard for Loki","editable":true,"fiscalYearStartMonth":0,"gnetId":13639,"graphTooltip":0,"id":3,"iteration":1662120358161,"links":[{"$$hashKey":"object:59","icon":"bolt","includeVars":true,"keepTime":true,"tags":[],"targetBlank":true,"title":"View In Explore","type":"link","url":"/explore?orgId=1&left=[\"now-1h\",\"now\",\"Loki\",{\"expr\":\"{job=\\\"$app\\\"}\"},{\"ui\":[true,true,true,\"none\"]}]"},{"$$hashKey":"object:61","icon":"external link","tags":[],"targetBlank":true,"title":"Learn LogQL","type":"link","url":"https://grafana.com/docs/loki/latest/logql/"}],"liveNow":false,"panels":[{"aliasColors":{},"bars":true,"dashLength":10,"dashes":false,"datasource":{"type":"loki","uid":"P8E80F9AEF21F6940"},"fieldConfig":{"defaults":{"links":[]},"overrides":[]},"fill":1,"fillGradient":0,"gridPos":{"h":3,"w":24,"x":0,"y":0},"hiddenSeries":false,"id":6,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":false,"total":false,"values":false},"lines":false,"linewidth":1,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"8.3.1","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(count_over_time({job=\"$app\"} |= \"$search\" [$__interval]))","legendFormat":"","refId":"A"}],"thresholds":[],"timeRegions":[],"tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:168","format":"short","logBase":1,"show":false},{"$$hashKey":"object:169","format":"short","logBase":1,"show":false}],"yaxis":{"align":false}},{"datasource":{"type":"loki","uid":"P8E80F9AEF21F6940"},"gridPos":{"h":25,"w":24,"x":0,"y":3},"id":2,"maxDataPoints":"","options":{"dedupStrategy":"none","enableLogDetails":true,"prettifyLogMessage":false,"showCommonLabels":false,"showLabels":false,"showTime":true,"sortOrder":"Ascending","wrapLogMessage":true},"targets":[{"datasource":{"type":"loki","uid":"P8E80F9AEF21F6940"},"expr":"{job=\"$app\",filename=\"$filename\"} |= \"$search\" ","hide":false,"legendFormat":"","refId":"A"}],"transparent":true,"type":"logs"}],"refresh":"30s","schemaVersion":33,"style":"dark","tags":[],"templating":{"list":[{"current":{"selected":true,"text":"test/raycluster-b-1565387256278454272","value":"test/raycluster-b-1565387256278454272"},"datasource":{"type":"loki","uid":"P8E80F9AEF21F6940"},"definition":"label_values(job)","hide":0,"includeAll":false,"label":"App","multi":false,"name":"app","options":[],"query":"label_values(job)","refresh":1,"regex":"","skipUrlSync":false,"sort":0,"tagValuesQuery":"","tagsQuery":"","type":"query","useTags":false},{"current":{"selected":true,"text":"/var/log/note_storage.log","value":"/var/log/note_storage.log"},"datasource":{"type":"loki","uid":"P8E80F9AEF21F6940"},"definition":"label_values({job=\"${app}\"},filename)","hide":0,"includeAll":false,"label":"FileName","multi":false,"name":"filename","options":[],"query":"label_values({job=\"${app}\"},filename)","refresh":1,"regex":"","skipUrlSync":false,"sort":0,"tagValuesQuery":"","tagsQuery":"","type":"query","useTags":false},{"current":{"selected":false,"text":"","value":""},"hide":0,"label":"String Match","name":"search","options":[{"selected":true,"text":"","value":""}],"query":"","skipUrlSync":false,"type":"textbox"}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"hidden":false,"refresh_intervals":["10s","30s","1m","5m","15m","30m","1h","2h","1d"]},"timezone":"","title":"Logs","uid":"4mP3nzG4z","version":2,"weekStart":""}
kind: ConfigMap
metadata:
creationTimestamp: null
name: grafana-dashboards
3. grafana deployment引用configmap
---
# Source: grafana/templates/deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: grafana
labels:
helm.sh/chart: grafana-6.18.2
app.kubernetes.io/name: grafana
app.kubernetes.io/instance: grafana
app.kubernetes.io/version: "8.3.1"
app.kubernetes.io/managed-by: Helm
spec:
replicas: 1
revisionHistoryLimit: 10
selector:
matchLabels:
app.kubernetes.io/name: grafana
app.kubernetes.io/instance: grafana
strategy:
type: RollingUpdate
template:
metadata:
labels:
app.kubernetes.io/name: grafana
app.kubernetes.io/instance: grafana
app: grafana
sidecar.istio.io/inject: "false"
annotations:
checksum/config: b09dc3addb625f8b5744b94f7ab624a3b02fdfd16aad7e4294a5fbf3608cb129
checksum/dashboards-json-config: 01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b
checksum/sc-dashboard-provider-config: 01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b
spec:
nodeSelector:
{{ .Values.global.k8s.daemonSet.workerLabelKey }}: "{{ .Values.global.k8s.daemonSet.workerLabelValue }}"
serviceAccountName: grafana
automountServiceAccountToken: true
securityContext:
fsGroup: 472
runAsGroup: 472
runAsUser: 472
enableServiceLinks: true
containers:
- name: grafana
image: "{{ .Values.global.repo.prefix }}/grafana:8.3.1"
imagePullPolicy: {{ .Values.global.repo.policy }}
volumeMounts:
- name: config
mountPath: "/etc/grafana/grafana.ini"
subPath: grafana.ini
- name: storage
mountPath: "/var/lib/grafana"
- name: grafana-dashboards
mountPath: "/var/lib/grafana/dashboards/model/model.json"
subPath: "model.json"
- name: grafana-dashboards
mountPath: "/var/lib/grafana/dashboards/logs/logs.json"
subPath: "logs.json"
- name: config
mountPath: "/etc/grafana/provisioning/datasources/datasources.yaml"
subPath: "datasources.yaml"
- name: config
mountPath: "/etc/grafana/provisioning/dashboards/dashboardproviders.yaml"
subPath: "dashboardproviders.yaml"
ports:
- name: service
containerPort: 3000
protocol: TCP
- name: grafana
containerPort: 3000
protocol: TCP
env:
- name: GF_PATHS_DATA
value: /var/lib/grafana/
- name: GF_PATHS_LOGS
value: /var/log/grafana
- name: GF_PATHS_PLUGINS
value: /var/lib/grafana/plugins
- name: GF_PATHS_PROVISIONING
value: /etc/grafana/provisioning
- name: "GF_AUTH_ANONYMOUS_ENABLED"
value: "false"
- name: "GF_AUTH_ANONYMOUS_ORG_ROLE"
value: "Admin"
- name: "GF_AUTH_BASIC_ENABLED"
value: "true"
- name: "GF_SECURITY_ADMIN_PASSWORD"
value: "1234qwer"
- name: "GF_SECURITY_ADMIN_USER"
value: "admin"
livenessProbe:
failureThreshold: 10
httpGet:
path: /api/health
port: 3000
initialDelaySeconds: 60
timeoutSeconds: 30
readinessProbe:
httpGet:
path: /api/health
port: 3000
resources:
{}
volumes:
- name: config
configMap:
name: grafana
- name: grafana-dashboards
configMap:
name: grafana-dashboards
- name: storage
emptyDir: {}