filebeat镜像制作
vim docker-entrypoint.sh
#!/bin/bash
ENV=${ENV:-"test"}
PROJ_NAME=${PROJ_NAME:-"no-define"}
MULTILINE=${MULTILINE:-"^\d{2}"}
cat > /etc/filebeat.yaml << EOF
filebeat.inputs:
- type: log
fields_under_root: true
fields:
topic: logm-${PROJ_NAME}
paths:
- /logm/*.log
- /logm/*/*.log
- /logm/*/*/*.log
- /logm/*/*/*/*.log
- /logm/*/*/*/*/*.log
scan_frequency: 120s
max_bytes: 10485760
multiline.pattern: '$MULTILINE'
multiline.negate: true
multiline.match: after
multiline.max_lines: 100
- type: log
fields_under_root: true
fields:
topic: logu-${PROJ_NAME}
paths:
- /logu/*.log
- /logu/*/*.log
- /logu/*/*/*.log
- /logu/*/*/*/*.log
- /logu/*/*/*/*/*.log
- /logu/*/*/*/*/*/*.log
output.kafka:
hosts: ["192.168.51.45:9092","192.168.51.46:9092","192.168.51.47:9092"]
topic: k8s-$ENV-%{[topic]}
version: 2.0.0
required_acks: 0
max_message_bytes: 10485760
EOF
set -xe
# If user don't provide any command
# Run filebeat
if [[ "$1" == "" ]]; then
exec filebeat -c /etc/filebeat.yaml
else
# Else allow the user to run arbitrarily commands like bash
exec "$@"
fi
vim Dockerfile
FROM centos:7
ENV FILEBEAT_VERSION=7.4.1
RUN set -x && \
curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-7.repo && \
yum install -y wget && \
wget https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-${FILEBEAT_VERSION}-linux-x86_64.tar.gz -O /opt/filebeat.tar.gz && \
cd /opt && \
tar xzvf filebeat.tar.gz && \
cd filebeat-* && \
cp filebeat /bin/ && \
cd /opt && \
rm -rf filebeat* && \
yum clean all
COPY docker-entrypoint.sh /
ENTRYPOINT ["/docker-entrypoint.sh"]
logstash配置
vim /etc/logstash/conf.d/k8s-pro.conf
input {
kafka {
bootstrap_servers => "192.168.51.45:9092,192.168.51.46:9092,192.168.51.47:9092"
consumer_threads => 4
group_id => "k8s_pro"
topics_pattern => "k8s-pro-.*"
}
}
filter {
json {
source => "message"
}
}
output {
elasticsearch {
hosts => ["192.168.51.45:9200","192.168.51.46:9200","192.168.51.47:9200"]
index => "k8s-pro-%{+YYYY.MM.DD}"
}
}
pod收集日志配置
vim deploy.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: nginx-xia
namespace: xia
spec:
replicas: 1
selector:
matchLabels:
appname: nginx-xia
template:
metadata:
labels:
appname: nginx-xia
spec:
affinity:
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- podAffinityTerm:
labelSelector:
matchExpressions:
- key: appname
operator: In
values:
- nginx-xia
topologyKey: kubernetes.io/hostname
weight: 50
containers:
- name: nginx-xia
image: harbor.com.cn/library/nginx:alpine-base
ports:
- containerPort: 80
volumeMounts:
- mountPath: /var/log/nginx
name: logu
readinessProbe:
failureThreshold: 3
httpGet:
path: /
port: 80
scheme: HTTP
initialDelaySeconds: 10
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
resources:
requests:
cpu: 200m
memory: 512Mi
limits:
cpu: 200m
memory: 512Mi
- name: filebeat
image: harbor.com.cn/library/filebeat:v7.4.1
imagePullPolicy: IfNotPresent
env:
- name: ENV
value: pro
- name: PROJ_NAME
value: nginx-xia
volumeMounts:
- mountPath: /logu
name: logu
volumes:
- emptyDir: {}
name: logu
---
apiVersion: v1
kind: Service
metadata:
annotations:
blackbox_path: "/"
blackbox_port: "80"
blackbox_scheme: "http"
name: nginx-xia
namespace: xia
spec:
selector:
appname: nginx-xia
ports:
- port: 80
protocol: TCP
targetPort: 80
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: nginx-xia
namespace: xia
annotations:
kubernetes.io/ingress.class: "nginx"
nginx.ingress.kubernetes.io/proxy-body-size: "1000m"
spec:
rules:
- host: xia.wuxingge.com
http:
paths:
- path: /
pathType: ImplementationSpecific
backend:
service:
name: nginx-xia
port:
number: 80