kubernetes-node方式日志收集(filebeat->kafka)

version: filebeat:6.2.4

1. 准备filebeat 配置文件：

$ cat filebeat.yml 
filebeat.registry_file: /var/log/containers/filebeat_registry

filebeat.prospectors:
-
  paths:
    - "/var/log/containers/*.log"

  symlinks: true
  json.message_key: log
  json.keys_under_root: true
  json.add_error_key: true

output.kafka:
  hosts: ${KAFKA_URL}
  topic: '${TOPIC_NAME}'
  partition.round_robin:
    reachable_only: false
  required_acks: ${ACKS:1}
  compression: gzip
  max_message_bytes: 1000000
  codec.format:
    string: '%{[log]}'

logging.level: ${LOG_LEVEL:error}

2. 编写Dockerfile:

$ cat Dockerfile 
FROM debian:jessie
ENV FILEBEAT_VERSION 6.2.4
RUN apt-get update && \
    apt-get -y install wget && \
    wget https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-${FILEBEAT_VERSION}-linux-x86_64.tar.gz && \
    tar xzf filebeat-${FILEBEAT_VERSION}-linux-x86_64.tar.gz && \
    mv filebeat-${FILEBEAT_VERSION}-linux-x86_64/filebeat /usr/local/bin && \
    rm -rf /filebeat* && \
    apt-get -y remove wget && \
    apt-get -y autoremove
COPY filebeat.yml /etc/filebeat/
RUN chmod go-w /etc/filebeat/filebeat.yml
CMD ["/usr/local/bin/filebeat", "-e", "-c", "/etc/filebeat/filebeat.yml"]

3. build 一个filebeat容器:

docker build --tag test/filebeat-kafka:latest .

4. 以daemonSet方式，在每个节点运行一个filebeat容器：

$ cat filebeat_ds.yaml 
apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
  name: filebeat
spec:
  template:
    metadata:
      labels:
        app: filebeat
        id: filebeat
      name: filebeat
    spec:
      containers:
      - name: filebeat-kafka
        image: test/filebeat-kafka:latest
        imagePullPolicy: IfNotPresent
        resources:
          limits:
            cpu: 50m
            memory: 50Mi
        env:
          - name: KAFKA_URL
            value: "[\"192.168.0.190:9092\"]"
          - name: TOPIC_NAME
            value: "k8s-logs"
          - name: ACKS
            value: "1"
          - name: LOG_LEVEL
            value: "info"
          - name: FILEBEAT_HOST
            valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
        volumeMounts:
        - name: varlog
          mountPath: /var/log/containers
        - name: varlogpods
          mountPath: /var/log/pods
          readOnly: true
        - name: varlibdockercontainers
          mountPath: /monitor-data/docker/containers
          readOnly: true
      terminationGracePeriodSeconds: 30
      tolerations:
      - key: node-role.kubernetes.io/master
        effect: NoSchedule
      volumes:
      - name: varlog
        hostPath:
          path: /var/log/containers
      - name: varlogpods
        hostPath:
          path: /var/log/pods
      - name: varlibdockercontainers
        hostPath:
          path: /monitor-data/docker/containers

 

参考：https://github.com/ApsOps/filebeat-kubernetes