logstash grok

最新推荐文章于 2023-07-08 01:23:58 发布

地球没有花

最新推荐文章于 2023-07-08 01:23:58 发布

阅读量435

点赞数

分类专栏：一句话文章标签： logstash grok

本文链接：https://blog.csdn.net/getyouwant/article/details/79011441

版权

一句话专栏收录该内容

55 篇文章 0 订阅

订阅专栏

利器。debug地址：

https://grokdebug.herokuapp.com/

https://github.com/elastic/logstash/blob/v1.4.0/patterns/grok-patterns

日志例子：

1.2.3.43 - - [11/Jan/2018:13:18:27 +0800] "GET /vcs/list?sso_tk=103XXXNGAa5hfwkaTWqxIm13jaaAMgq5YQTI8fOONPkrMm1YEOOVm5Cm1VCiW41qqMa1Em5NVm2ARZuVUYcm4&type=3,4&page=1&pagesize=20&btime=30&lang=&takeaway=null HTTP/1.1" 200 857 "-" "Apache-HttpClient/4.5.1 (Java/1.7.0_51)" "H:api.met,r:123.125.38.97,le:123.125.38.67,xf:123.135.58.67,ag:-,reqid:1515647907.692-31842-18168418022" 518 0.005 0.005

上面的日志是根据nginx定义的格式生成的：

$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" "H:$http_host,r:$http_x_real_ip,le:$http_leproxy_forwarded_for,xf:$http_x_forwarded_for,ag:$arg_ip,reqid:$req_id" $request_length $request_time $upstream_response_time'

$remote_addr：1.2.3.43

$remote_user：-

$time_local：11/Jan/2018:13:18:27 +0800

$request：GET /vcs/list?sso_tk=103XXXNGAa5hfwkaTWqxIm13jaaAMgq5YQTI8fOONPkrMm1YEOOVm5Cm1VCiW41qqMa1Em5NVm2ARZuVUYcm4&type=3,4&page=1&pagesize=20&btime=30&lang=&takeaway=null HTTP/1.1

$status：200

$body_bytes_sent：857

$http_referer：-

$http_user_agent：Apache-HttpClient/4.5.1 (Java/1.7.0_51)

$http_host：api.met

$http_x_real_ip：123.125.38.97

$http_leproxy_forwarded_for：123.135.58.67

$http_x_forwarded_for：123.135.58.67

$arg_ip：-

$req_id：1515647907.692-31842-18168418022

$request_length：518

$request_time：0.005

$upstream_response_time：0.005

下面是grok match pattern：

%{IP:remote_addr} - %{USERNAME:remote_user} \[%{HTTPDATE:time_local}\] \"%{GREEDYDATA:request_method} %{GREEDYDATA:request_api} %{GREEDYDATA:http_verion}\" %{NUMBER:status} %{NUMBER:body_bytes_sent} %{QS:http_referer} %{QS:http_user_agent} \"H:%{HOSTNAME:http_host},r:%{IP:http_x_real_ip},le:%{IP:http_leproxy_forwarded_for},xf:%{IP:http_x_forwarded_for},ag:%{GREEDYDATA:arg_ip},reqid:%{USERNAME:req_id}\" %{NUMBER:request_length} %{NUMBER:request_time} %{NUMBER:upstream_response_time}

解析结果如下：

{
  "remote_addr": [
    [
      "1.2.3.43"
    ]
  ],
  "IPV6": [
    [
      null,
      null,
      null,
      null
    ]
  ],
  "IPV4": [
    [
      "1.2.3.43",
      "123.125.38.97",
      "123.125.38.67",
      "123.135.58.67"
    ]
  ],
  "remote_user": [
    [
      "-"
    ]
  ],
  "time_local": [
    [
      "11/Jan/2018:13:18:27 +0800"
    ]
  ],
  "MONTHDAY": [
    [
      "11"
    ]
  ],
  "MONTH": [
    [
      "Jan"
    ]
  ],
  "YEAR": [
    [
      "2018"
    ]
  ],
  "TIME": [
    [
      "13:18:27"
    ]
  ],
  "HOUR": [
    [
      "13"
    ]
  ],
  "MINUTE": [
    [
      "18"
    ]
  ],
  "SECOND": [
    [
      "27"
    ]
  ],
  "INT": [
    [
      "+0800"
    ]
  ],
  "request_method": [
    [
      "GET"
    ]
  ],
  "request_api": [
    [
      "/vcs/list?sso_tk=103XXXNGAa5hfwkaTWqxIm13jaaAMgq5YQTI8fOONPkrMm1YEOOVm5Cm1VCiW41qqMa1Em5NVm2ARZuVUYcm4&type=3,4&page=1&pagesize=20&btime=30&lang=&takeaway=null"
    ]
  ],
  "http_verion": [
    [
      "HTTP/1.1"
    ]
  ],
  "status": [
    [
      "200"
    ]
  ],
  "BASE10NUM": [
    [
      "200",
      "857",
      "518",
      "0.005",
      "0.005"
    ]
  ],
  "body_bytes_sent": [
    [
      "857"
    ]
  ],
  "http_referer": [
    [
      ""-""
    ]
  ],
  "QUOTEDSTRING": [
    [
      ""-"",
      ""Apache-HttpClient/4.5.1 (Java/1.7.0_51)""
    ]
  ],
  "http_user_agent": [
    [
      ""Apache-HttpClient/4.5.1 (Java/1.7.0_51)""
    ]
  ],
  "http_host": [
    [
      "api.met"
    ]
  ],
  "http_x_real_ip": [
    [
      "123.125.38.97"
    ]
  ],
  "http_leproxy_forwarded_for": [
    [
      "123.125.38.67"
    ]
  ],
  "http_x_forwarded_for": [
    [
      "123.135.58.67"
    ]
  ],
  "arg_ip": [
    [
      "-"
    ]
  ],
  "req_id": [
    [
      "1515647907.692-31842-18168418022"
    ]
  ],
  "request_length": [
    [
      "518"
    ]
  ],
  "request_time": [
    [
      "0.005"
    ]
  ],
  "upstream_response_time": [
    [
      "0.005"
    ]
  ]
}