rsyslog+filebeat

##rsyslog

#### RULES ####
$template DynaFile,"/data/log/net/other/%FROMHOST-IP%_%$YEAR%-%$MONTH%-%$DAY%"
$template myformat,"%$NOW% %TIMESTAMP:8:15% %hostname% %syslogtag% %msg%\n"
$ActionFileDefaultTemplate myformat

$template hq,"/data/log/net/HQ/%FROMHOST-IP%_%$YEAR%-%$MONTH%-%$DAY%"
 

:fromhost-ip, isequal, "127.0.0.1" ?Local
& ~
:hostname,contains,"CC" ?cc
& ~
:hostname,startswith,"HQ" ?hq
& ~
:syslogtag,startswith,"HQ" ?hq
& ~

:FROMHOST-IP,startswith,"10.32.4." ?hq
& ~
:FROMHOST-IP,startswith,"10.32.5" ?hq
& ~

*.*                                                     -?DynaFile
& ~

##filebeat

cat /etc/yum.repos.d/filebeat.repo
[filebeat]

name=Elasticsearch repository for 88888888.x packages

baseurl=https://artifacts.elastic.co/packages/8.x/yum

gpgcheck=0

gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch

enabled=1

autorefresh=1

type=rpm-md

#

 yum install filebeat -y 

#

##3、filebeat 配置

filebeat.inputs:

- type: log
  enabled: true
  paths:
    - /data/log/other/10.1.1.1*
    #- /data/log/net/FW/10.1.1.1*
  fields:
      device_model: "zb-caiwuwww-92"
  #    kafka_topic: "zb-zhuanxianfw-1-1-1-1-topic"

filebeat.config.modules:
  path: ${path.config}/modules.d/*.yml
  reload.enabled: false

setup.template.settings:
  index.number_of_shards: 1

setup.template.enabled: true
setup.template.fields: fields.yml
setup.template.overwrite: true

processors:
- drop_fields:
     fields: ['agent', 'ecs', 'beat', 'input_type', 'tags', 'count', '@version', 'log', 'offset', 'type', 'host']
     ignore_missing: false

#输出到logstash

output.logstash:
  hosts: ["10.1.1.1:5044"]

output.kafka:
    enabled: true
    hosts: ["10.1.1.1:9092","10.1.1.2:9092","10.1.1.3:9092"]
    #hosts: ["10.1.1.1:9092"]
    topic: "%{[fields.kafka_topic]}"
    compression: gzip
    max_message_bytes: 1000000