filebeat-cm.yaml
# cat filebeat-cm.yaml
---
apiVersion: v1
kind: ConfigMap
metadata:
name: filebeat-config
namespace: kube-system
labels:
k8s-app: filebeat
data:
filebeat.yml: |-
setup.ilm.enabled: false
scan_frequency: 10s
ignore_older: 3h
clean_inactive: 6h
filebeat.inputs:
- type: container
containers.ids:
- "*"
paths:
- /var/log/containers/*.log
max_bytes: 102400
processors:
- add_kubernetes_metadata:
default_indexers.enabled: true
default_matchers.enabled: true
in_cluster: true
host: ${NODE_NAME}
matchers:
- logs_path:
logs_path: "/var/log/containers/"
- drop_fields:
fields: ["host", "kubernetes.node.labels", "tags", "ecs", "prospector", "agent", "input", "beat"]
ignore_missing: true
- fingerprint:
fields: ["log.offset", "log.file.path"]
target_field: "@metadata._id"
multiline.pattern: '^[[:space:]]+(at|\.{3})\b|^Caused by:'
multiline.negate: false
multiline.match: after
setup.template.name: "k8s"
setup.template.pattern: "k8s-*"
setup.template.enabled: false
setup.template.overwrite: false
setup.template.settings:
index.number_of_shards: 2
index.number_of_replicas: 0
output.elasticsearch:
hosts: ['${ELASTICSEARCH_HOST:elasticsearch}:${ELASTICSEARCH_PORT:9200}']
username: ${ELASTICSEARCH_USERNAME}
password: ${ELASTICSEARCH_PASSWORD}
worker: 20
max_retries: 3
bulk_max_size: 800
#index: "k8s-%{[kubernetes.namespace]}-%{[kubernetes.container.name]}-%{+yyyy.MM.dd}"
indices:
# - index: "k8s-%{[kubernetes.namespace]}-%{[kubernetes.container.name]}-%{+yyyy.MM.dd}"
- index: "k8s-%{[kubernetes.namespace]}"
setup.kibana:
host: ':'
# 设置 ilm 的 policy life, 日志保留 45 天, 每 7 天一轮训, 最大 30g
setup.ilm:
policy_file: /etc/indice-lifecycle.json
---
apiVersion: v1
kind: ConfigMap
metadata:
name: filebeat-index-rules
namespace: kube-system
labels:
k8s-app: filebeat
data:
indice-lifecycle.json: |-
{
"policy": {
"phases": {
"hot": {
"actions": {
"rollover": {
"max_size": "5GB" ,
"max_age": "1d"
}
}
},
"delete": {
"min_age": "7d",
"actions": {
"delete": {}
}
}
}
}
}
filebeat-ds
# cat filebeat-ds.yaml
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: filebeat
namespace: kube-system
labels:
k8s-app: filebeat
spec:
selector:
matchLabels:
k8s-app: filebeat
template:
metadata:
labels:
k8s-app: filebeat
spec:
hostNetwork: true
dnsPolicy: "ClusterFirstWithHostNet"
serviceAccountName: filebeat
terminationGracePeriodSeconds: 30
tolerations:
- effect: NoSchedule
operator: Exists
- effect: NoExecute
operator: Exists
containers:
- name: filebeat
image: elastic/filebeat:7.13.1
args: [
"-c", "/etc/filebeat.yml",
"-e",
]
env:
- name: ELASTICSEARCH_PORT
value: "9200"
- name: ELASTICSEARCH_HOSTS
value: '["http://elasticsearch-logging:9200"]'
- name: ELASTICSEARCH_USERNAME
value: elastic
- name: ELASTICSEARCH_PASSWORD
value: elastic
- name: ELASTIC_CLOUD_ID
value:
- name: ELASTIC_CLOUD_AUTH
value:
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
securityContext:
runAsUser: 0
resources:
requests:
cpu: 100m
memory: 500Mi
volumeMounts:
- name: config
mountPath: /etc/filebeat.yml
readOnly: true
subPath: filebeat.yml
- name: rules
mountPath: /etc/indice-lifecycle.json
readOnly: true
subPath: indice-lifecycle.json
- name: data
mountPath: /usr/share/filebeat/data
- name: varlibdockercontainers
mountPath: /var/lib/docker/containers
readOnly: true
- name: varlog
mountPath: /var/log
readOnly: true
volumes:
- name: config
configMap:
defaultMode: 0600
name: filebeat-config
- name: rules
configMap:
defaultMode: 0600
name: filebeat-index-rules
- name: varlibdockercontainers
hostPath:
path: /var/log/pods
- name: varlog
hostPath:
path: /var/log
# data folder stores a registry of read status for all files, so we don't send everything again on a Filebeat pod restart
- name: data
hostPath:
path: /var/lib/filebeat-data
type: DirectoryOrCreate
rbac
# cat rbac.yaml
---
# Source: filebeat/templates/filebeat-service-account.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: filebeat
namespace: kube-system
labels:
k8s-app: filebeat
---
# Source: filebeat/templates/filebeat-role.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: filebeat
labels:
k8s-app: filebeat
rules:
- apiGroups: [""] # "" indicates the core API group
resources:
- namespaces
- pods
- nodes
verbs:
- get
- watch
- list
---
# Source: filebeat/templates/filebeat-role-binding.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: filebeat
subjects:
- kind: ServiceAccount
name: filebeat
namespace: kube-system
roleRef:
kind: ClusterRole
name: filebeat
apiGroup: rbac.authorization.k8s.io