系列文章目录
实现原理
一个java副本程序里包含一个 filebeat 程序,java程序固定输入到目录日志,filebeat 进行采集,推送到 es,kibala 进行采集
kind: Deployment
apiVersion: apps/v1
metadata:
labels:
app: $IMAGES
name: $IMAGES
namespace: yanxuan
spec:
progressDeadlineSeconds: 600
replicas: 1
selector:
matchLabels:
app: $IMAGES
strategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 25%
maxSurge: 25%
template:
metadata:
labels:
app: $IMAGES
spec:
imagePullSecrets:
- name: aliyun-docker-hub
containers:
- image: 'elastic/filebeat:7.17.3'
args: [
"-c","/etc/filebeat.yml",
"-e",
]
name: filebeat
resources:
limits:
cpu: '500'
memory: 500Mi
requests:
cpu: '0.1'
memory: 100Mi
volumeMounts:
- name: $IMAGES
mountPath: /data/logs #挂载收集日志目录
- name: filebeat-config #挂载 filebeat ConfigMap的日志收集脚本
mountPath: /etc/filebeat.yml
subPath: filebeat.yml #指定子路径
- image: '$REGISTRY/$DOCKERHUB_NAMESPACE/$IMAGES:$BUILD_NUMBER' #后端java 服务
name: $IMAGES
ports:
- containerPort: $JAR_PORD
protocol: TCP
resources: #这里是最大使用
limits:
cpu: '2.5'
memory: 2500Mi
requests: #这里是最小使用
cpu: '0.1'
memory: 256Mi
volumeMounts: #将java日志输出到 挂载目录下/data/logs
- name: $IMAGES
mountPath: /data/logs
livenessProbe: #健康检查
initialDelaySeconds: 60
timeoutSeconds: 10
periodSeconds: 10
successThreshold: 1
failureThreshold: 5
httpGet:
scheme: HTTP
path: $INSPECT
port: $JAR_PORD
volumes:
- name: $IMAGES
emptyDir: {}
- name: filebeat-config
configMap:
name: filebeat-config
terminationGracePeriodSeconds: 30
---
apiVersion: v1
kind: ConfigMap
metadata:
namespace: yanxuan
name: filebeat-config
data:
filebeat.yml: |- #配置字典、日志采集格式
filebeat.inputs:
- type: log
paths:
- /data/logs/console.log #获取的logs
fields:
type: "$IMAGES"
multiline:
pattern: '^202.*' #分割 以时间开头分割
negate: true
match: after
setup.ilm.enabled: false
setup.template.name: "log"
setup.template.pattern: "log-*"
output.elasticsearch:
hosts: ["$ES:9200"]
indices:
- index: "$IMAGES-%{+yyyy.MM.dd}"
when.equals:
fields.type: "$IMAGES"
setup.kibana:
host: "$ES:5601"
processors:
- script:
lang: javascript
id: my_filter
tag: enable
source: >
function process(event) {
var message= event.Get("message");
var time =message.split("|")[0];
event.Put("log_time",time);
message= message.replace(time, '')
event.Put("message", message);
var message= event.Get("message");
var time =message.split("|")[3];
event.Put("ERROR",time);
}
- timestamp:
field: log_time
timezone: Asia/Shanghai
layouts:
- '2006-01-02 15:04:05'
- '2006-01-02 15:04:05.999'
test:
- '2019-06-22 16:33:51'