ELK集群部署(九)之logstash过滤规则

logstash过滤规则

标准输入
定制过滤规则
标准输出到屏幕

test.yml模板
input{
   stdin {}
}

filter{
   mutate{xxxxxx} # 不同的过滤需求，需要替换这里的内容
}

output{
   stdout {
      codec => rubydebug
   }
}

====================================================================

grok{“message” => “\s+(?<request_time>\d+(?.\d+)?)\s+”}

结果

{
        "@version" => "1",
         "message" => "hello 23132.4245435 Jason",
            "host" => "node-2",
      "@timestamp" => 2022-05-20T05:49:55.054Z,
          "second" => ".4245435",
    "request_time" => "23132.4245435"
}

grok{“message” => “%{WORD} %{NUMBER:request_time:float} %{WORD}”}
结果

{
            "host" => "node-2",
        "@version" => "1",
    "request_time" => 123.456,
      "@timestamp" => 2022-05-20T05:55:14.774Z,
         "message" => "begin 123.456 end"
}

分割字符串

mutate{ split => ["message","|"]}
输入123|345|jason|long|darcy

结果
{
       "message" => [
        [0] "123",
        [1] "345",
        [2] "jason",
        [3] "long"，
        [4] "darcy"
    ],
          "host" => "node-2",
      "@version" => "1",
    "@timestamp" => 2022-05-20T06:36:46.494Z
}

join只对数组类型字段有效，先分割，替换分隔符再合并

mutate{ split => [“message”,“|”]}
mutate{ join => [“message”,“,”]}
输入123|345|jason|long|darcy
结果

{
       "message" => "123,345,jason,long,darcy",
          "host" => "node-2",
      "@version" => "1",
    "@timestamp" => 2022-05-20T06:41:19.607Z
}

字符串拼接

mutate{ split => [“message”,“|”]}
mutate{ merge => [“message”,“host”]}
输入123|345|jason|long|darcy
结果

{
          "host" => "node-2",
       "message" => [
        [0] "123",
        [1] "345",
        [2] "jason",
        [3] "long",
        [4] "darcy",
        [5] "node-2"
    ],
      "@version" => "1",
    "@timestamp" => 2022-05-20T06:48:01.814Z
}

strip去除字段前后的空格

mutate{strip => [“message”]}
输入 jason 注意前后都有多个空格

结果

{
          "host" => "node-2",
      "@version" => "1",
       "message" => "jason",
    "@timestamp" => 2022-05-20T06:54:02.590Z
}

rename字段重命名

mutate{ rename => [“host”, “newhost”]}
输入 jason 注意前后都有多个空格
结果

{
       "message" => "jason",
    "@timestamp" => 2022-05-20T06:57:07.478Z,
      "@version" => "1",
       "newhost" => "node-2"
}

split一行拆分成多个事件

split{field=>“message” terminator=>“#”}
输入test1#test2

{
    "@timestamp" => 2022-05-20T07:14:19.414Z,
          "host" => "node-2",
       "message" => "test1",
      "@version" => "1"
}
{
    "@timestamp" => 2022-05-20T07:14:19.414Z,
          "host" => "node-2",
       "message" => "test2",
      "@version" => "1"
}

交叉日志合并多行日志

 filter {
   grok {
     match => [ "message", "%{LOGLEVEL:loglevel} - %{NOTSPACE:taskid} - %{NOTSPACE:logger} - %{WORD:label}( - %{INT:duration:int})?" ]
   }

   if [logger] == "TASK_START" {
     aggregate {
       task_id => "%{taskid}"
       code => "map['sql_duration'] = 0"
       map_action => "create"
     }
   }

   if [logger] == "SQL" {
     aggregate {
       task_id => "%{taskid}"
       code => "map['sql_duration'] += event.get('duration')"
       map_action => "update"
     }
   }

   if [logger] == "TASK_END" {
     aggregate {
       task_id => "%{taskid}"
       code => "event.set('sql_duration', map['sql_duration'])"
       map_action => "update"
       end_of_task => true
       timeout => 120
     }
   }
 }

依次输入
INFO - 12345 - TASK_START - start
INFO - 12345 - SQL - sqlQuery1 - 12
INFO - 12345 - SQL - sqlQuery2 - 34
INFO - 12345 - TASK_END - end

结果

{
          "label" => "end",
   "sql_duration" => 46,
           "host" => "node-2",
       "loglevel" => "INFO",
         "taskid" => "12345",
         "logger" => "TASK_END",
     "@timestamp" => 2022-05-20T07:43:34.820Z,
       "@version" => "1",
        "message" => "INFO - 12345 - TASK_END - end"
}

参考官网https://www.elastic.co/guide/en/logstash/current/plugins-filters-aggregate.html

发送邮件

input{
   stdin {}
}

output{
   stdout {
      codec => rubydebug
   }
   email{
    to => "jasonxxx@foxmail.com"
    from => "logstash@qq.com"
    subject => "Alert - %{+YYYY-MM-dd HH:mm:ss}"
    body => "Tags: Test\n\nContent:\n%{message}"
    port => 25
  }

}

直接用上面的模板，logstash就可以发送邮件