nginx日志 - filebeat - kafka - logstash - elasticsearch - kibana

elasticsearch-7.7.1
filebeat-7.7.1-linux-x86_64
kafka_2.13-2.5.0
kibana-7.7.1-linux-x86_64
logstash-7.7.1

参照：
各种组件安装：https://blog.51cto.com/xiangcun168/1933509
单机版kafka+zookeeper：https://blog.51cto.com/xiangcun168/1933375
Nginx日志格式、反向代理配置：https://blog.csdn.net/stephen_curry11/article/details/85339898

1、启动elasticsearch
如果出现max virtual memory areas vm.max_map_count，或者出现 the default discovery settings are unsuitable for production use; at least one of [discovery.seed_hosts, discovery.seed_providers, cluster.initial_master_nodes] must be configured，参考 https://www.cnblogs.com/hellxz/p/11057234.html，执行：sudo sysctl -w vm.max_map_count=262144
启动：nohup bin/elasticsearch >> nohup.es &
验证：http://192.168.2.99:9200/

2、启动kibana
启动：nohup bin/kibana >> nohup.kibana &
验证：http://192.168.2.99:5601/

3、启动zookeeper
启动：nohup bin/zookeeper-server-start.sh config/zookeeper.properties >> nohup.zookeeper &
验证：bin/zookeeper-shell.sh 192.168.2.99:2181 ls /

4、启动kafka
启动：nohup bin/kafka-server-start.sh config/server.properties >> nohup.kafka &
如果报clusterID不匹配，/config/server.properties找到logs.dir/meta.properties，修改cluster.id即可
验证：

bin/kafka-topics.sh --create --bootstrap-server 192.168.2.99:9092 --replication-factor 1 --partitions 1 --topic test
bin/kafka-topics.sh --list --bootstrap-server 192.168.2.99:9092
bin/kafka-topics.sh --list --zookeeper 192.168.2.99:2181

bin/kafka-console-producer.sh --broker-list 192.168.2.99:9092 --topic test
bin/kafka-console-consumer.sh --bootstrap-server 192.168.2.99:9092 --topic test --from-beginning

5、启动nginx
修改日志格式（nginx.conf）：

    log_format  json  '{"@version":"1",'
                      '"time_local":"$time_local",'
                      '"remote_addr":"$remote_addr",'
                      '"http_host":"$http_host",'
                      '"http_x_forwarded_for":"$http_x_forwarded_for",'
                      '"request_method":"$request_method",'
                      '"request_uri":"$request_uri",'
                      '"server_protocol":"$server_protocol",'
                      '"status":"$status",'
                      '"body_bytes_sent":"$body_bytes_sent",'
                      '"http_referer":"$http_referer",'
                      '"http_user_agent":"$http_user_agent",'
                      '"request_time":"$request_time",'
                      '"upstream_response_time":"$upstream_response_time"'
                      '}';
    access_log  logs/access.log  json;

输出：

{
	"@version": "1",
	"time_local": "16/Jul/2020:14:16:25 +0800",
	"remote_addr": "192.168.2.103",
	"http_host": "192.168.2.99:7000",
	"http_x_forwarded_for": "-",
	"request_method": "GET",
	"request_uri": "/",
	"server_protocol": "HTTP/1.1",
	"status": "304",
	"body_bytes_sent": "0",
	"http_referer": "-",
	"http_user_agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36",
	"request_time": "0.000",
	"upstream_response_time": "-"
}

启动：sudo /usr/local/nginx/sbin/nginx
验证：http://192.168.2.99:7000/

6、启动filebeat
启动：nohup ./filebeat -c filebeat.yml >> nohup.filebeat &
验证：bin/kafka-console-consumer.sh --bootstrap-server 192.168.2.99:9092 --topic ngx-access --from-beginning --group grp1
结果：

{
	"@timestamp": "2020-07-16T06:16:27.801Z",
	"@metadata": {
		"beat": "filebeat",
		"type": "_doc",
		"version": "7.7.1"
	},
	"log": {
		"offset": 3685504,
		"file": {
			"path": "/usr/local/nginx/logs/access.log"
		}
	},
	"message": "{\"@version\":\"1\",\"time_local\":\"16/Jul/2020:14:16:25 +0800\",\"remote_addr\":\"192.168.2.103\",\"http_host\":\"192.168.2.99:7000\",\"http_x_forwarded_for\":\"-\",\"request_method\":\"GET\",\"request_uri\":\"/\",\"server_protocol\":\"HTTP/1.1\",\"status\":\"304\",\"body_bytes_sent\":\"0\",\"http_referer\":\"-\",\"http_user_agent\":\"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36\",\"request_time\":\"0.000\",\"upstream_response_time\":\"-\"}",
	"input": {
		"type": "log"
	},
	"log_type": ["ngx_access"],
	"agent": {
		"type": "filebeat",
		"ephemeral_id": "71135a05-4022-4b9a-b32b-0543c269dfb4",
		"hostname": "ebda-Z390-UD",
		"id": "a9a2827c-98cc-46cd-9869-360f17946787",
		"version": "7.7.1"
	},
	"tags": ["ngx"],
	"log_topic": "ngx-access",
	"ecs": {
		"version": "1.5.0"
	},
	"host": {
		"name": "ebda-Z390-UD"
	}
}

7、启动logstah
配置（nginx.conf）：

input {
    kafka  {
        codec => "json"
        topics_pattern => "ngx-access"
        bootstrap_servers => "192.168.2.99:9092"
        auto_offset_reset => "latest"
        group_id => "logstash-ngx-access"
    }
}

filter {
    if "ngx" in [tags] {
      json {
        source => "message"
        target => "msg"
        skip_on_invalid_json => true
      }
    }
}

output {

    elasticsearch {
        hosts => ["192.168.2.99:9200"]
        index => "logstash-%{type}-%{+YYYY.MM.dd}"
        #document_type => "%{type}"
    }

#stdout { codec => rubydebug }
}

1）document_type需要设置为_doc或去掉，否则会提示Rejecting mapping update to [house] as the final mapping would have more than 1 type: [_doc, XXXX]
2）如果json中没有target，所有字段会和message字段平级
启动：nohup bin/logstash -f config/nginx.conf >> nohup.logstash &

8、kibana呈现
Discover中看到的日志：

{
  "_index": "logstash-ngx-access-2020.07.16",
  "_type": "_doc",
  "_id": "JatXVnMBG8J699ywC3FX",
  "_version": 1,
  "_score": null,
  "_source": {
    "ecs": {
      "version": "1.5.0"
    },
    "host": {
      "name": "ebda-Z390-UD"
    },
    "@version": "1",
    "log_type": [
      "ngx_access"
    ],
    "@timestamp": "2020-07-16T06:36:50.614Z",
    "log_topic": "ngx-access",
    "msg": {
      "http_user_agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36",
      "@version": "1",
      "request_time": "0.000",
      "http_referer": "-",
      "upstream_response_time": "-",
      "body_bytes_sent": "0",
      "remote_addr": "192.168.2.103",
      "time_local": "16/Jul/2020:14:36:47 +0800",
      "server_protocol": "HTTP/1.1",
      "request_uri": "/",
      "http_x_forwarded_for": "-",
      "status": "304",
      "http_host": "192.168.2.99:7000",
      "request_method": "GET"
    },
    "message": "{\"@version\":\"1\",\"time_local\":\"16/Jul/2020:14:36:47 +0800\",\"remote_addr\":\"192.168.2.103\",\"http_host\":\"192.168.2.99:7000\",\"http_x_forwarded_for\":\"-\",\"request_method\":\"GET\",\"request_uri\":\"/\",\"server_protocol\":\"HTTP/1.1\",\"status\":\"304\",\"body_bytes_sent\":\"0\",\"http_referer\":\"-\",\"http_user_agent\":\"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36\",\"request_time\":\"0.000\",\"upstream_response_time\":\"-\"}",
    "agent": {
      "type": "filebeat",
      "ephemeral_id": "71135a05-4022-4b9a-b32b-0543c269dfb4",
      "version": "7.7.1",
      "hostname": "ebda-Z390-UD",
      "id": "a9a2827c-98cc-46cd-9869-360f17946787"
    },
    "input": {
      "type": "log"
    },
    "tags": [
      "ngx"
    ],
    "log": {
      "offset": 3687367,
      "file": {
        "path": "/usr/local/nginx/logs/access.log"
      }
    }
  },
  "fields": {
    "@timestamp": [
      "2020-07-16T06:36:50.614Z"
    ]
  },
  "sort": [
    1594881410614
  ]
}