ELK-logstash--写入到kafka和从kafka读取日志

logstash写入到kafka和从kafka读取日志

收集nginx日志放到kafka

input {
  file {
    type => "nginx-access"
    path => "/data/wwwlogs/access_nginx.log"
    start_position => "beginning"
    codec => json
 }
 
  file {
   path => "/var/log/messages"
   start_position => "beginning"
   type => "system-log-252"
    }
   }
 }
output {
    if [type] == "nginx-access" {
kafka {
    bootstrap_servers => "192.168.1.252:9092" #kafka服务器地址
    topic_id => "252nginx-accesslog"
    batch_size => 5
    codec => "json" #写入的时候使用json编码，因为logstash收集后会转换成json格式
      }
      } 
                }
  if [type] == "system-log-252" {
 kafka {
  bootstrap_servers => "192.168.1.252:9092"
  topic_id => "system-log-252"
  batch_size => 5
  codec => "json" #写入的时候使用json编码，因为logstash收集后会转换成json格式
             }
                 }
     }
}

配置logstash从kafka读取日志

input {
  kafka {
     bootstrap_servers => "192.168.1.252:9092" #kafka服务器地址
     topics => "252nginx-accesslog"
     batch_size => 5
     codec => "json" #写入的时候使用json编码，因为logstash收集后会转换成json格式
     group_id => "252nginx-access-log" 
     consumer_threads => 1
     decorate_events => true 
  }
         kafka {
bootstrap_servers => "192.168.1.252:9092"
topics => "system-log-252"
consumer_threads => 1
decorate_events => true
codec => "json" 
}
}
output {
  if [type] == "252nginx-accesslo" {
  elasticsearch {
    hosts => ["192.168.1.252:9200"]
    index => "252nginx-accesslog-%{+YYYY.MM.dd}"
  }}
          if [type] == "system-log-252" {
 elasticsearch {
   hosts => ["192.168.1.252:9200"]
   index => "system-log-1512-%{+YYYY.MM.dd}"
 }
} 

[root@elk-node01 apache-tomcat-8.5.39]# cat conf/server.xml
	  <Valve className="org.apache.catalina.valves.AccessLogValve" directory="logs"
           prefix="tomcat_access_log" suffix=".log"
           pattern="{&quot;clientip&quot;:&quot;%h&quot;,&quot;ClientUser&quot;:&quot;%l&quot;,&quot;authenticated&quot;:&quot;%u&quot;,&quot;AccessTime&quot;:&quot;%t&quot;,&quot;method&quot;:&quot;%r&quot;,&quot;status&quot;:&quot;%s&quot;,&quot;SendBytes&quot;:&quot;%b&quot;,&quot;Query?string&quot;:&quot;%q&quot;,&quot;partner&quot;:&quot;%{Referer}i&quot;,&quot;AgentVersion&quot;:&quot;%{User-Agent}i&quot;}" />