version: filebeat:6.2.4
1. 准备filebeat 配置文件:
$ cat filebeat.yml
filebeat.registry_file: /var/log/containers/filebeat_registry
filebeat.prospectors:
-
paths:
- "/var/log/containers/*.log"
symlinks: true
json.message_key: log
json.keys_under_root: true
json.add_error_key: true
output.kafka:
hosts: ${KAFKA_URL}
topic: '${TOPIC_NAME}'
partition.round_robin:
reachable_only: false
required_acks: ${ACKS:1}
compression: gzip
max_message_bytes: 1000000
codec.format:
string: '%{[log]}'
logging.level: ${LOG_LEVEL:error}
2. 编写Dockerfile:
$ cat Dockerfile
FROM debian:jessie
ENV FILEBEAT_VERSION 6.2.4
RUN apt-get update && \
apt-get -y install wget && \
wget https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-${FILEBEAT_VERSION}-linux-x86_64.tar.gz && \
tar xzf filebeat-${FILEBEAT_VERSION}-linux-x86_64.tar.gz && \
mv filebeat-${FILEBEAT_VERSION}-linux-x86_64/filebeat /usr/local/bin && \
rm -rf /filebeat* && \
apt-get -y remove wget && \
apt-get -y autoremove
COPY filebeat.yml /etc/filebeat/
RUN chmod go-w /etc/filebeat/filebeat.yml
CMD ["/usr/local/bin/filebeat", "-e", "-c", "/etc/filebeat/filebeat.yml"]
3. build 一个filebeat容器:
docker build --tag test/filebeat-kafka:latest .
4. 以daemonSet方式,在每个节点运行一个filebeat容器:
$ cat filebeat_ds.yaml
apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
name: filebeat
spec:
template:
metadata:
labels:
app: filebeat
id: filebeat
name: filebeat
spec:
containers:
- name: filebeat-kafka
image: test/filebeat-kafka:latest
imagePullPolicy: IfNotPresent
resources:
limits:
cpu: 50m
memory: 50Mi
env:
- name: KAFKA_URL
value: "[\"192.168.0.190:9092\"]"
- name: TOPIC_NAME
value: "k8s-logs"
- name: ACKS
value: "1"
- name: LOG_LEVEL
value: "info"
- name: FILEBEAT_HOST
valueFrom:
fieldRef:
fieldPath: spec.nodeName
volumeMounts:
- name: varlog
mountPath: /var/log/containers
- name: varlogpods
mountPath: /var/log/pods
readOnly: true
- name: varlibdockercontainers
mountPath: /monitor-data/docker/containers
readOnly: true
terminationGracePeriodSeconds: 30
tolerations:
- key: node-role.kubernetes.io/master
effect: NoSchedule
volumes:
- name: varlog
hostPath:
path: /var/log/containers
- name: varlogpods
hostPath:
path: /var/log/pods
- name: varlibdockercontainers
hostPath:
path: /monitor-data/docker/containers
参考:https://github.com/ApsOps/filebeat-kubernetes