logstash + grok 正则语法

详细正则规则参考:

正则语法规则

例:

日志格式如下

[vclound][2015-11-03 03:35:50,283][INFO][/usr/lib/python2.6/site-packages/urllib3/connectionpool.py:203][_new_conn][-][140192616544000]=[Starting new HTTP connection (1): 240.10.129.80]
[vclound][2015-11-03 03:35:50,381][DEBUG][/usr/lib/python2.6/site-packages/urllib3/connectionpool.py:295][_make_request][-][140192616544000]=["POST /v2.0/tokens HTTP/1.1" 200 3080]
[vclound][2015-11-03 03:35:50,384][INFO][/usr/lib/python2.6/site-packages/urllib3/connectionpool.py:203][_new_conn][-][140192616544000]=[Starting new HTTP connection (1): 240.10.129.160]
[vclound][2015-11-03 03:35:50,454][DEBUG][/usr/lib/python2.6/site-packages/urllib3/connectionpool.py:295][_make_request][-][140192616544000]=["GET /v2/bb0b51d166254dc99bc7462c0ac002ff/servers/b4b530e7-cd9b-42c1-bcd4-a48140726846 HTTP/1.1" 404 73]

logstash 正则规则参考

(下面代码, 编辑器无法显示, 请点击 view plain 进行阅读)

filter {
  if [type] == "pinyun" {
    grok {
      match => { "message" => "\[%{USERNAME:username}\]\[%{TIMESTAMP_ISO8601:time}\]\[%{LOGLEVEL:loglevel}\]\[%{PROG:filepath}\]\[%{PROG:function}\]\[-\]\[%{BASE16NUM:progid}\]\=\[%{GREEDYDATA:info}\]" }
      add_field => [ "received_at", "%{@timestamp}" ]
      add_field => [ "received_from", "%{host}" ]
    }
  }
}

注意: 当日志输出有空格, 那么匹配时候就带空格, 如果是特殊字符, 那么就直接匹配该特殊字符

输出效果

如下:

{
          "message" => "[vclound][2015-11-03 03:35:50,283][INFO][/usr/lib/python2.6/site-packages/urllib3/connectionpool.py:203][_new_conn][-][140192616544000]=[Starting new HTTP connection (1): 240.10.129.80]",
         "@version" => "1",
       "@timestamp" => "2015-11-03T02:01:30.051Z",
             "type" => "pinyun",
             "file" => "/apps/logs/uwsgi/uwsgi.log",
             "host" => "terry-zskvt.vclound.com",
           "offset" => "58995",
         "username" => "vclound",
             "time" => "2015-11-03 03:35:50,283",
         "loglevel" => "INFO",
         "filepath" => "/usr/lib/python2.6/site-packages/urllib3/connectionpool.py:203",
         "function" => "_new_conn",
           "progid" => "140192616544000",
             "info" => "Starting new HTTP connection (1): 240.10.129.80",
      "received_at" => "2015-11-03T02:01:30.051Z",
    "received_from" => "terry-zskvt.vclound.com"
}
{
          "message" => "[vclound][2015-11-03 03:35:50,381][DEBUG][/usr/lib/python2.6/site-packages/urllib3/connectionpool.py:295][_make_request][-][140192616544000]=[\"POST /v2.0/tokens HTTP/1.1\" 200 3080]",
         "@version" => "1",
       "@timestamp" => "2015-11-03T02:01:30.060Z",
             "type" => "pinyun",
             "file" => "/apps/logs/uwsgi/uwsgi.log",
             "host" => "terry-zskvt.vclound.com",
           "offset" => "59181",
         "username" => "vclound",
             "time" => "2015-11-03 03:35:50,381",
         "loglevel" => "DEBUG",
         "filepath" => "/usr/lib/python2.6/site-packages/urllib3/connectionpool.py:295",
         "function" => "_make_request",
           "progid" => "140192616544000",
             "info" => "\"POST /v2.0/tokens HTTP/1.1\" 200 3080",
      "received_at" => "2015-11-03T02:01:30.060Z",
    "received_from" => "terry-zskvt.vclound.com"
}
{
          "message" => "[vclound][2015-11-03 03:35:50,384][INFO][/usr/lib/python2.6/site-packages/urllib3/connectionpool.py:203][_new_conn][-][140192616544000]=[Starting new HTTP connection (1): 240.10.129.160]",
         "@version" => "1",
       "@timestamp" => "2015-11-03T02:01:30.068Z",
             "type" => "pinyun",
             "file" => "/apps/logs/uwsgi/uwsgi.log",
             "host" => "terry-zskvt.vclound.com",
           "offset" => "59362",
         "username" => "vclound",
             "time" => "2015-11-03 03:35:50,384",
         "loglevel" => "INFO",
         "filepath" => "/usr/lib/python2.6/site-packages/urllib3/connectionpool.py:203",
         "function" => "_new_conn",
           "progid" => "140192616544000",
             "info" => "Starting new HTTP connection (1): 240.10.129.160",
      "received_at" => "2015-11-03T02:01:30.068Z",
    "received_from" => "terry-zskvt.vclound.com"
}
{
          "message" => "[vclound][2015-11-03 03:35:50,454][DEBUG][/usr/lib/python2.6/site-packages/urllib3/connectionpool.py:295][_make_request][-][140192616544000]=[\"GET /v2/bb0b51d166254dc99bc7462c0ac002ff/servers/b4b530e7-cd9b-42c1-bcd4-a48140726846 HTTP/1.1\" 404 73]",
         "@version" => "1",
       "@timestamp" => "2015-11-03T02:01:30.074Z",
             "type" => "pinyun",
             "file" => "/apps/logs/uwsgi/uwsgi.log",
             "host" => "terry-zskvt.vclound.com",
           "offset" => "59549",
         "username" => "vclound",
             "time" => "2015-11-03 03:35:50,454",
         "loglevel" => "DEBUG",
         "filepath" => "/usr/lib/python2.6/site-packages/urllib3/connectionpool.py:295",
         "function" => "_make_request",
           "progid" => "140192616544000",
             "info" => "\"GET /v2/bb0b51d166254dc99bc7462c0ac002ff/servers/b4b530e7-cd9b-42c1-bcd4-a48140726846 HTTP/1.1\" 404 73",
      "received_at" => "2015-11-03T02:01:30.074Z",
    "received_from" => "terry-zskvt.vclound.com"
}