1、日志到filebeat。 cat /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: syslog
format: rfc3164
protocol.udp:
host: "0.0.0.0:514"
output.logstash:
hosts: ["localhost:5044"]
验证方式: tcpdump -i 网卡名称 udp port 514
2、logstash
docker run -d \
--name=logstash_xx \
--restart=always \
-p 5044:5044 \
-p 9600:9600 \
-v /data/logstash_xx/pipeline/logstash.conf:/usr/share/logstash/pipeline/logstash.conf \
docker.elastic.co/logstash/logstash:8.10.3
docker run -d \
--name=logstash_fw_16.5-t2 \
--restart=always \
-p 15045:5044/udp \
-p 15045:5044 \
-p 19605:9600 \
-v /data/logstash_fw_16.5/config/logstash.yml:/usr/share/logstash/config/logstash.ymlh \
-v /data/logstash_fw_16.5/pipeline/logstash.conf:/usr/share/logstash/pipeline/logstash.conf \
-v /data/logstash_fw_16.5/config/jvm.options:/usr/share/logstash/config/jvm.options \
docker.elastic.co/logstash/logstash:8.10.3
##
cat /data/logstash_fw_16.5/config/logstash.yml
http.host: "0.0.0.0"
xpack.monitoring.elasticsearch.hosts: ["esip1:9200","esip2:9200","esip3:9200" ]
xpack.monitoring.enabled: true
xpack.monitoring.elasticsearch.username: elastic
xpack.monitoring.elasticsearch.password: 123qwe
##
# 监听从kafka发过来的数据
input {
# 有多少个kafka topic 下面就需要重复多少个,同类型的可以写到一个topic里面
kafka {
type => "zb-zhuanxianfw-10-84-16-5"
topics => "zb-zhuanxianfw-10-84-16-5-topic"
bootstrap_servers => "kafka_ip1:9092,"kafka_ip2:9092,"kafka_ip3:9092"
auto_offset_reset => "latest"
codec => json
}
}
filter {
if [fields][device_model] == "zb-zhuanxianfw-10-84-16-5"{
grok {
match => { "message" => "(?<时间>\d{1,4}-\d{1,2}-\d{1,2}\s{1,2}\d{1,2}:\d{1,2}:\d{1,2}) (?<设备名称>[^ ]+) (?<日志类型>[^:]+): (?<系统>[^,]+), (?<协议>[^,]+), (?<源地址>[^,]+), (?<源端口>[^,]+), (?<目的地址>[^,]+), (?<目的端口>[^,]+), (?<时间2>[^,]+), (?<源zone>[^,]+), (?<目的zone>[^,]+), (?<应用名>[^,]+), (?<策略名称>[^.]+)."}
}
}
####end
else {}
# 修改时间 增加8小时
#ruby {
# code => "event.set('timestamp', event.get('@timestamp').time.localtime + 8*60*60)"
# }
# mutate {
# remove_field => ["@timestamp"]
# }
# 删除下面这些标签信息
mutate {
remove_field => "agent"
remove_field => "ecs"
remove_field => "@version"
remove_field => "host"
remove_field => "path"
remove_field => "log.offset"
remove_field => "tags"
remove_field => "input"
remove_field => "offset"
remove_field => "_score"
}
}
# 打印到屏幕
#output {
# stdout {}
# }
output {
elasticsearch {
user =>"elastic"
password =>"123qwe"
hosts => ["esip1:9200","esip2:9200","esip3:9200"]
index => "%{[fields][device_model]}-%{+YYYY.MM.dd}"
}
}
##
cat /data/logstash_fw_16.5/config/jvm.options
-Xms4g
-Xmx32g
11-13:-XX:+UseConcMarkSweepGC
11-13:-XX:CMSInitiatingOccupancyFraction=75
11-13:-XX:+UseCMSInitiatingOccupancyOnly
-Djava.awt.headless=true
-Dfile.encoding=UTF-8
-Djruby.compile.invokedynamic=true
-XX:+HeapDumpOnOutOfMemoryError
-Djava.security.egd=file:/dev/urandom
-Dlog4j2.isThreadContextMapInheritable=true
查看logstash 状态: http://logstash_ip:9600/_node/pipelines?pretty
2.1 logstash 配置如下:logstash.conf
input {
beats {
port => 5044
host => "0.0.0.0" # 或者 "localhost"
}
}
filter {
if "SZZB-A2F_C06-30U-USG6550_FMS-02_xx" in [message] {
mutate {
add_field => { "logfrom" => "huawei-log" }
}
}
else if "10.84.x.x" in [log][source][address] {
mutate {
add_field => { "logfrom" => "caiwu_huawei" }
}
}
else {
mutate {
add_field => { "logfrom" => "other" }
}
}
if [logfrom] == "huawei-log" {
grok {
match => [
"message", "<%{BASE10NUM:syslog_pri}>%logstash正则表达式*)"
]
}
}
}
##
output {
stdout {}
}
output {
elasticsearch {
user => "elastic"
password => "密码"
hosts => ["https://xxxip:9200"]
ssl_certificate_verification => false
index => "%{[logfrom]}-%{+YYYY.MM.dd}"
}
}
2.3 验证方式:
tcpdump -i docker0 tcp port 5044 #docker网卡里面会有数据,说明logstash已经把数据发出来了
kibana配置:可用