FileBeat7.7读取nginx access.log写入kafka

$ vim filebeat.yml

filebeat.inputs:
- type: log
  paths:
    - /usr/local/nginx/logs/access.log
  fields:
    log_type: ["ngx_access"]
    log_topic: ngx-access
  fields_under_root: true
  tail_files: true

tags: ["ngx"]

output.kafka:
  enabled: true
  hosts: ["192.168.2.99:9092"]
  topic: '%{[log_topic]}'

写入kafka的消息：

{
	"@timestamp": "2020-07-09T07:12:31.680Z",
	"@metadata": {
		"beat": "filebeat",
		"type": "_doc",
		"version": "7.7.1"
	},
	"log_type": ["ngx_access"],
	"tags": ["ngx"],
	"input": {
		"type": "log"
	},
	"log_topic": "ngx-access",
	"agent": {
		"ephemeral_id": "37b744ad-f553-45b5-9128-5e320f2c34dd",
		"hostname": "ebda-Z390-UD",
		"id": "a9a2827c-98cc-46cd-9869-360f17946787",
		"version": "7.7.1",
		"type": "filebeat"
	},
	"ecs": {
		"version": "1.5.0"
	},
	"host": {
		"name": "ebda-Z390-UD"
	},
	"log": {
		"offset": 37302,
		"file": {
			"path": "/usr/local/nginx/logs/access.log"
		}
	},
	"message": "192.168.2.105 - - [09/Jul/2020:15:12:26 +0800] \"GET / HTTP/1.1\" 304 0 \"-\" \"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36\""
}

将 fields_under_root: true 去掉时，fields中的字段输出变化：

	"fields": {
		"log_topic": "ngx-access",
		"log_type": ["ngx_access"]
	},

同时，取值时需要这样写：

topic: '%{[fields.log_topic]}'