input
详解参考之前的文章
https://blog.csdn.net/gekkoou/article/details/80986017
input
官方详解
https://www.elastic.co/guide/en/logstash/current/input-plugins.html
filter
官方详解
https://www.elastic.co/guide/en/logstash/current/filter-plugins.html
output
官方详解
https://www.elastic.co/guide/en/logstash/current/output-plugins.html
贴上收集 apache
日志的代码做简介
input {
# access日志
file {
type => "apache_access"
tag => "apache_access"
path => ["/var/log/apache/access.log"]
start_position => beginning
}
# error日志
file {
type => "apache_error"
tag => "apache_error"
path => ["/var/log/apache/error.log"]
start_position => beginning
}
}
filter {
# 根据 input 添加的 type 来区分, 实现同时读取两种日志, 也可以用 tag 来区分 (例如 if [tag] in "apache_access")
if [type] == "apache_access"{
# 文本片段切分的方式来切分日志事件
# 推荐使用grokdebugger来写匹配模式: http://grokdebug.herokuapp.com/
# grok官方详解: https://www.elastic.co/guide/en/logstash/current/plugins-filters-grok.html
grok {
match => { "message" => "%{COMBINEDAPACHELOG}"}
}
# data插件可以用来转换你的日志记录中的时间字符串, 然后转存到 @timestamp 字段里
date {
match => [ "timestamp" , "dd/MMM/YYYY:HH:mm:ss Z" ]
}
# 通过geoip能获取到很多的信息,包括经纬度,国家,城市,地区等信息
geoip {
# 来源于apache日志中的clientip
source => "clientip"
}
# useragent插件可以帮助我们过滤出浏览器版本、型号以及系统版本
useragent {
source => "agent"
target => "useragent"
}
} else if [type] == "apache_error"{
grok {
match => { "message" => "\[(?<mytimestamp>%{DAY:day} %{MONTH:month} %{MONTHDAY} %{TIME} %{YEAR})\] \[%{WORD:module}:%{LOGLEVEL:loglevel}\] \[pid %{NUMBER:pid}:tid %{NUMBER:tid}\]( \(%{POSINT:proxy_errorcode}\)%{DATA:proxy_errormessage}:)?( \[client %{IPORHOST:client}:%{POSINT:clientport}\])? %{DATA:errorcode}: %{GREEDYDATA:message}" }
}
date {
match => [ "mytimestamp" , "EEE MMM dd HH:mm:ss.SSSSSS yyyy" ]
}
}
#转换类型 (integer, float, integer_eu, float_eu, string, boolean)
#mutate {
# convert => ["ctime", "integer"]
# convert => ["lat", "float"]
#}
#当某条日志信息符合if规则时
#if [field_name] == "value" {
# #drop可以跳过某些不想统计的日志信息
# drop {}
#}
#create_at为时间戳时需要转换为0时区(UTC), 然后放入@timestamp字段里
#date {
# match => ["create_at", "yyyy-MM-dd HH:mm:ss,SSS", "UNIX"]
# #match => ["create_at", "UNIX"]
# target => "@timestamp"
# locale => "cn"
# #remove_field => 'create_at' #删除字段
#}
# 执行ruby代码
#ruby {
# code => "event.set('timestamp', event.get('@timestamp').time.localtime + 8*60*60)"
#}
#ruby {
# code => "event.set('@timestamp',event.get('timestamp'))"
#}
}
# 输出插件将数据发送到一个特定的目的地, 除了elasticsearch还有好多可输出的地方, 例如file, csv, mongodb, redis, syslog等
output {
if [type] == "apache_access"{
elasticsearch {
hosts => [ "localhost:9200" ]
# 记录的index索引名称格式
index => "apache-access-log-%{+YYYY.MM}"
}
} else if [type] == "apache_error"{
elasticsearch {
hosts => [ "localhost:9200" ]
index => "apache-error-log"
}
}
}