ELK之logstash配置

ELK之logstash配置

一、logstash直接监听日志文件`

input{
 file {
	path => "/tmp/logs/info/gateway.log"
	start_position => "beginning"
	type => "log1"
   }
 file {
	path => "/tmp/logs/debug/boot.log"
	start_position => "beginning"
	type => "log2"
   }
}
output{
	 if[type] == "log1"{
	  elasticsearch {
		   hosts => ["192.167.213.11:9100"]      
		   index => "gateway-%{+YYYY.MM.dd}"        
		}
	}
	if[type] == "log2"{
	  elasticsearch {
		   hosts => ["192.168.213.11:9100"]      
		   index => "boot-%{+YYYY.MM.dd}"        
		}
	}	
}

二、filebeat整合logstash

filebeat.yml

# filebeat.yml
filebeat.prospectors:
- type: log
  paths:
    - /tmp/logs/info/gate.log
  input_type: log
  tags: ["log1"]
  fields:
    type: gate
  document_type: gate
  multiline.pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
  multiline.negate: true
  multiline.match: after
  
 - type: log
  paths:
   -/tmp/logs/debug/boot.log
  input_type: log
  tags: ["log2"]
  fields:
    type: boot
  document_type: boot
  multiline.pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
  multiline.negate: true
  multiline.match: after
  
fields:
  env: develop-203.11
output:
  logstash:
    hosts: ["192.168.203.11:5000"]
    

注：filebeat的文件由于是yml 所以语法严谨一些
logstash.conf

	input{
		beats {
		port => 5000
		codec => "json"
	}
	}	
	filter {
	grok {
		match  => {
			"message" => "(?<date>(\d*[./-]\d*[./-]\d* \d*:\d*:\d*[.,][0-9]+)) .*%{LOGLEVEL:level} .*\- \[%{USERNAME:requestId}\] \[%{HOSTNAME:ip}\] \[%{USERNAME:channel}\] \[%{USERNAME:serviceName}\] \[%{INT:spendTime}\ms] \[%{INT:code}]"
		}
		overwrite => ["message"]		
	}
	mutate {
		convert => ["spendTime", "integer"]
	}
}
	output{		
		 if "log1" in [tags]{
		  elasticsearch {
			   hosts => ["192.168.213.11:9200"]      
			   index => "gate-%{+YYYY.MM.dd}"        
			}
		}
		if "log2" in [tags]{
		  elasticsearch {
			   hosts => ["192.168.213.11:9200"]      
			   index => "boot-%{+YYYY.MM.dd}"        
			}
		}		
	}