filebeat收集kubernetes集群日志

1、filebeat结合本地es服务收集kubernetes日志

[root@kubernetes-master ~]# cat filebeat-kubernetes.yaml
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: filebeatconf
  namespace: kube-system
  labels:
    k8s-app: filebeat
data:
  filebeat.yml: |-
    filebeat.inputs:
    - type: container
      paths:
        - /var/log/containers/*.log
      fields:
        pod_name: '${podName}'
        pod_ip: '${podIp}'
        pod_namespace: '${podNamespace}'
      processors:
        - add_kubernetes_metadata:
            host: ${NODE_NAME}
            matchers:
            - logs_path:
                logs_path: "/var/log/containers/"

    # To enable hints based autodiscover, remove `filebeat.inputs` configuration and uncomment this:
    #filebeat.autodiscover:
    #  providers:
    #    - type: kubernetes
    #      node: ${NODE_NAME}
    #      hints.enabled: true
    #      hints.default_config:
    #        type: container
    #        paths:
    #          - /var/log/containers/*${data.kubernetes.container.id}.log

    processors:
      - add_cloud_metadata:
      - add_host_metadata:

    cloud.id: ${ELASTIC_CLOUD_ID}
    cloud.auth: ${ELASTIC_CLOUD_AUTH}

    output.elasticsearch:
      hosts: ['192.168.122.182:9200']    # 集群外部署的es集群
    output.elasticsearch.index: "phl-filebeat-%{+yyyy.MM.dd}"    # 自定义索引
    setup.template.name: "phl-filebeat"
    setup.template.pattern: "phl-filebeat-%{+yyyy.MM.dd}"
    setup.ilm.enabled: false       # 关闭lim，不关闭不能使用自定义索引

---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: filebeat
  namespace: kube-system
  labels:
    k8s-app: filebeat
spec:
  selector:
    matchLabels:
      k8s-app: filebeat
  template:
    metadata:
      labels:
        k8s-app: filebeat
    spec:
      serviceAccountName: filebeat
      terminationGracePeriodSeconds: 30
      hostNetwork: true
      dnsPolicy: ClusterFirstWithHostNet
      containers:
      - name: filebeat
        image: 192.168.122.150/library/filebeat:7.6.2
        env:
        - name: NODE_NAME
          valueFrom:
            fieldRef:
              fieldPath: spec.nodeName
        - name: podIp
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: status.podIP
        - name: podName
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: metadata.name
        - name: podNamespace
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: metadata.namespace
        securityContext:
          runAsUser: 0
          # If using Red Hat OpenShift uncomment this:
          #privileged: true
        resources:
          limits:
            memory: 200Mi
          requests:
            cpu: 100m
            memory: 100Mi
        volumeMounts:
        - name: data
          mountPath: /usr/share/filebeat/data
        - name: varlibdockercontainers
          mountPath: /var/lib/docker/containers
          readOnly: true
        - name: varlog
          mountPath: /var/log
        - name: filebeatconf
          mountPath: /usr/share/filebeat/filebeat.yml 
          subPath: usr/share/filebeat/filebeat.yml
          readOnly: true
      volumes:
      - name: varlibdockercontainers
        hostPath:
          path: /var/lib/docker/containers
      - name: varlog
        hostPath:
          path: /var/log
      - name: filebeatconf
        configMap:
          name: filebeatconf
          items:
            - key: filebeat.yml
              path: usr/share/filebeat/filebeat.yml

      # data folder stores a registry of read status for all files, so we don't send everything again on a Filebeat pod restart
      - name: data
        hostPath:
          # When filebeat runs as non-root user, this directory needs to be writable by group (g+w).
          path: /var/lib/filebeat-data
          type: DirectoryOrCreate
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: filebeat
subjects:
- kind: ServiceAccount
  name: filebeat
  namespace: kube-system
roleRef:
  kind: ClusterRole
  name: filebeat
  apiGroup: rbac.authorization.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: filebeat
  labels:
    k8s-app: filebeat
rules:
- apiGroups: [""] # "" indicates the core API group
  resources:
  - namespaces
  - pods
  verbs:
  - get
  - watch
  - list
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: filebeat
  namespace: kube-system
  labels:
    k8s-app: filebeat
---

2、查看是否创建成功

[root@kubernetes-master ~]# kubectl get pod,cm -n kube-system
NAME                                            READY   STATUS    RESTARTS   AGE
pod/filebeat-btbq8                              1/1     Running   0          35m
pod/filebeat-cn4nt                              1/1     Running   0          35m

NAME                                           DATA   AGE
configmap/filebeatconf                         1      112m

3、kibana查看是否生成自定义索引


4、使用创建的自定义索引查询日志

5、上图可以看到使用自定义索引能够精确的找到对应服务日志