elk 之 logstash + kafka + Zookpeer + Elasticsearch + kibana

这里我们用kafka做中间件，安装看前几篇

 1,logstash把日志输入到kafka集群

[root@elk_120 conf.d]# cat nginx-kafka.conf 
input {
 file {
 path => "/var/log/nginx/access.log"
 start_position => "beginning"
 type => "nginx-accesslog-1512"
 codec => "json"
 }
}
output {
 if [type] == "nginx-accesslog-1512" {
 kafka {
 bootstrap_servers => "10.1.234.200:9092" 
 topic_id => "nginx-accesslog-1512"
 codec => "json"
 }
 file {
 path => "/tmp/nginx-jsog-log.txt"
 }
 }
}

2，logstash 从kafka 集群取出日志输入到es 集群

[root@elk_120 conf.d]# cat kafka-es.conf 
input {
 kafka {
 bootstrap_servers => "10.1.234.200:9092"
 topics => "nginx-accesslog-1512"
 codec => "json"
 consumer_threads => 1
 #decorate_events => true
 }
}
output {
 if [type] == "nginx-accesslog-1512" {
 elasticsearch {
 hosts => ["10.1.234.200:9200"]
 index => "nginx-accesslog-1512-%{+YYYY.MM.dd}"
 }}
}

3，检查配置文件

[root@elk_120 conf.d]#  /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/nginx-kafka.conf -t 
WARNING: Could not find logstash.yml which is typically located in $LS_HOME/config or /etc/logstash. You can specify the path using --path.settings. Continuing using the defaults
Could not find log4j2 configuration at path /usr/share/logstash/config/log4j2.properties. Using default config which logs errors to the console
[WARN ] 2020-08-24 22:25:42.373 [LogStash::Runner] multilocal - Ignoring the 'pipelines.yml' file because modules or command line options are specified
[INFO ] 2020-08-24 22:25:44.184 [LogStash::Runner] Reflections - Reflections took 51 ms to scan 1 urls, producing 20 keys and 40 values 
Configuration OK
[INFO ] 2020-08-24 22:25:44.770 [LogStash::Runner] runner - Using config.test_and_exit mode. Config Validation Result: OK. Exiting Logstash

4，启动服务

systemctl restart logstash

[root@elk_120 conf.d]# ss -nutlp | grep 9300
tcp    LISTEN     0      128     ::ffff:10.1.234.120:9300                 :::*                   users:(("java",pid=13851,fd=224))

 5，查看es 

curl -XGET 'http://10.1.234.200:9200/_cat/indices/?v'