这里我们用kafka做中间件,安装看前几篇
1,logstash把日志输入到kafka集群
[root@elk_120 conf.d]# cat nginx-kafka.conf
input {
file {
path => "/var/log/nginx/access.log"
start_position => "beginning"
type => "nginx-accesslog-1512"
codec => "json"
}
}
output {
if [type] == "nginx-accesslog-1512" {
kafka {
bootstrap_servers => "10.1.234.200:9092"
topic_id => "nginx-accesslog-1512"
codec => "json"
}
file {
path => "/tmp/nginx-jsog-log.txt"
}
}
}
2,logstash 从kafka 集群取出日志输入到es 集群
[root@elk_120 conf.d]# cat kafka-es.conf
input {
kafka {
bootstrap_servers => "10.1.234.200:9092"
topics => "nginx-accesslog-1512"
codec => "json"
consumer_threads => 1
#decorate_events => true
}
}
output {
if [type] == "nginx-accesslog-1512" {
elasticsearch {
hosts => ["10.1.234.200:9200"]
index => "nginx-accesslog-1512-%{+YYYY.MM.dd}"
}}
}
3,检查配置文件
[root@elk_120 conf.d]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/nginx-kafka.conf -t
WARNING: Could not find logstash.yml which is typically located in $LS_HOME/config or /etc/logstash. You can specify the path using --path.settings. Continuing using the defaults
Could not find log4j2 configuration at path /usr/share/logstash/config/log4j2.properties. Using default config which logs errors to the console
[WARN ] 2020-08-24 22:25:42.373 [LogStash::Runner] multilocal - Ignoring the 'pipelines.yml' file because modules or command line options are specified
[INFO ] 2020-08-24 22:25:44.184 [LogStash::Runner] Reflections - Reflections took 51 ms to scan 1 urls, producing 20 keys and 40 values
Configuration OK
[INFO ] 2020-08-24 22:25:44.770 [LogStash::Runner] runner - Using config.test_and_exit mode. Config Validation Result: OK. Exiting Logstash
4,启动服务
systemctl restart logstash
[root@elk_120 conf.d]# ss -nutlp | grep 9300
tcp LISTEN 0 128 ::ffff:10.1.234.120:9300 :::* users:(("java",pid=13851,fd=224))
5,查看es
curl -XGET 'http://10.1.234.200:9200/_cat/indices/?v'