logstash grok正则调试

原创 2016年08月29日 15:39:38
logstash 正则调试;
nginx 配置;
log_format  main  '$remote_addr [$time_local] "$request" ';


logstash:
 "message" =>"%{IPORHOST:clientip} \[%{HTTPDATE:time}\] \"%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}\""
 
 输出:
 
 {
        "message" => " 121.40.205.143 [29/Aug/2016:12:36:32 +0800] \"GET /favicon.ico HTTP/1.1\" - 404 2319 \"-\" \"Mozilla/5.0 (Linux; Android 5.1.1; vivo X6S A Build/LMY47V) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/37.0.0.0 Mobile MQQBrowser/6.2 TBS/036558 Safari/537.36 MicroMessenger/6.3.25.861 NetType/WIFI Language/zh_CN\" 0.000 -",
       "@version" => "1",
     "@timestamp" => "2016-08-29T04:39:16.608Z",
           "path" => "/rsyslog/data/nginx/uat/nginx_access01_log.2016-08-29",
           "host" => "0.0.0.0",
           "type" => "uat_nginx_access",
       "clientip" => "121.40.205.143",
           "time" => "29/Aug/2016:12:36:32 +0800",
           "verb" => "GET",
        "request" => "/favicon.ico",
    "httpversion" => "1.1"
}

此时grok 能正常匹配:

            "message" => "%{IPORHOST:clientip} \[%{HTTPDATE:time}\] \"%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}\" 
			%{NUMBER:http_status_code} %{NUMBER:bytes} \"(?<http_referer>\S+)\" \"(?<http_user_agent>\S+)\" \"(?<http_x_forwarded_for>\S+)\""
			
			
			    log_format  main  '$http_host $server_addr $remote_addr [$time_local] "$request" '
                      '$request_body $status $body_bytes_sent "$http_referer" "$http_user_agent" '
                      '$request_time $upstream_response_time';
					  
继续加;
    log_format  main  '$remote_addr [$time_local] "$request"'
                       '$status $body_bytes_sent';
					   
					  
日志格式:
 121.40.205.143 [29/Aug/2016:12:51:18 +0800] "GET /resources/plugins/artDialog/ui-dialog.css HTTP/1.1"304 0
 121.40.205.143 [29/Aug/2016:12:51:18 +0800] "GET /wechat/images/account/icons.7a340e21.png HTTP/1.1"304 0
 121.40.205.143 [29/Aug/2016:12:51:18 +0800] "GET /wechat/images/nav-icon.44c2022c.png?v=1 HTTP/1.1"304 0
 121.40.205.143 [29/Aug/2016:12:51:19 +0800] "GET /favicon.ico HTTP/1.1"404 2319
 121.40.205.143 [29/Aug/2016:12:51:19 +0800] "GET /favicon.ico HTTP/1.1"404 2319
 121.40.205.143 [29/Aug/2016:12:52:25 +0800] "GET /favicon.ico HTTP/1.1"404 2319
 121.40.205.143 [29/Aug/2016:12:52:25 +0800] "GET /favicon.ico HTTP/1.1"404 2319
 121.40.205.143 [29/Aug/2016:12:53:28 +0800] "GET /favicon.ico HTTP/1.1"404 2319
 
 
filter {
    grok {
        match=> {
       "message" =>"%{IPORHOST:clientip} \[%{HTTPDATE:time}\] \"%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}\"%{NUMBER:http_status_code} %{NUMBER:bytes}"
       
        }

		
logstash 输出:
Pipeline main started
{
             "message" => " 121.40.205.143 [29/Aug/2016:12:56:10 +0800] \"GET /favicon.ico HTTP/1.1\"404 2319",
            "@version" => "1",
          "@timestamp" => "2016-08-29T04:58:54.908Z",
                "path" => "/rsyslog/data/nginx/uat/nginx_access01_log.2016-08-29",
                "host" => "0.0.0.0",
                "type" => "uat_nginx_access",
            "clientip" => "121.40.205.143",
                "time" => "29/Aug/2016:12:56:10 +0800",
                "verb" => "GET",
             "request" => "/favicon.ico",
         "httpversion" => "1.1",
    "http_status_code" => "404",
               "bytes" => "2319"
}					   



继续;


 121.40.205.143 [29/Aug/2016:13:00:16 +0800] "GET /favicon.ico HTTP/1.1"404 2319 "-"
 121.40.205.143 [29/Aug/2016:13:00:22 +0800] "GET /favicon.ico HTTP/1.1"404 2319 "-"
 121.40.205.143 [29/Aug/2016:13:00:30 +0800] "GET /favicon.ico HTTP/1.1"404 2319 "-"
 121.40.205.143 [29/Aug/2016:13:00:32 +0800] "GET /wechat/login.html HTTP/1.1"304 0 "https://uatest.winfae.com/wechat/account.html"
 121.40.205.143 [29/Aug/2016:13:00:32 +0800] "GET /wechat/images/login/icon_01.6e839367.png HTTP/1.1"304 0 "https://uatest.winfae.com/wechat/css/wechat.2a00a782.css"
 121.40.205.143 [29/Aug/2016:13:00:32 +0800] "GET /wechat/images/login/icon_02.5065faba.png HTTP/1.1"304 0 "https://uatest.winfae.com/wechat/css/wechat.2a00a782.css"
 121.40.205.143 [29/Aug/2016:13:00:32 +0800] "GET /resources/plugins/jquery/jquery.md5.js?v=1 HTTP/1.1"304 0 "https://uatest.winfae.com/wechat/login.html"
 121.40.205.143 [29/Aug/2016:13:00:32 +0800] "GET /wechat/js/libs/dialog-min.88247f5e.js?v=1 HTTP/1.1"304 0 "https://uatest.winfae.com/wechat/login.html"
 121.40.205.143 [29/Aug/2016:13:00:32 +0800] "GET /wechat/js/login.a87fbd64.js HTTP/1.1"304 0 "https://uatest.winfae.com/wechat/login.html"


{
       "message" => " 121.40.205.143 [29/Aug/2016:13:05:24 +0800] \"GET /wechat/account_balance.html HTTP/1.1\"200 3059 \"https://uatest.winfae.com/wechat/account.html\" 
	   \"Mozilla/5.0 (Linux; Android 5.1.1; vivo X6S A Build/LMY47V) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/37.0.0.0 Mobile MQQBrowser/6.2 TBS/036558 Safari/537.36 MicroMessenger/6.3.25.861 NetType/WIFI Language/zh_CN\"",

121.40.205.143 [29/Aug/2016:13:05:24 +0800] "GET /wechat/account_balance.html HTTP/1.1"200 3059 "https://uatest.winfae.com/wechat/account.html" "Mozilla/5.0 (Linux; Android 5.1.1; vivo X6S A Build/LMY47V) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/37.0.0.0 Mobile MQQBrowser/6.2 TBS/036558 Safari/537.36 MicroMessenger/6.3.25.861 NetType/WIFI Language/zh_CN"
121.40.205.143 [29/Aug/2016:13:05:45 +0800] "GET /wechat/home.html?useragent=android_h5_zjcap&apiver=2 HTTP/1.1"200 11601 "-" "okhttp/2.6.0"



{
             "message" => " 121.40.205.143 [29/Aug/2016:13:13:11 +0800] \"GET /wechat/js/regain.431efde9.js HTTP/1.1\"304 0 \"https://uatest.winfae.com/wechat/regain.html\" \"Mozilla/5.0 (Linux; Android 5.1.1; vivo X6S A Build/LMY47V) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/37.0.0.0 Mobile MQQBrowser/6.2 TBS/036558 Safari/537.36 MicroMessenger/6.3.25.861 NetType/WIFI Language/zh_CN\"",
            "@version" => "1",
          "@timestamp" => "2016-08-29T05:15:55.609Z",
                "path" => "/rsyslog/data/nginx/uat/nginx_access01_log.2016-08-29",
                "host" => "0.0.0.0",
                "type" => "uat_nginx_access",
            "clientip" => "121.40.205.143",
                "time" => "29/Aug/2016:13:13:11 +0800",
                "verb" => "GET",
             "request" => "/wechat/js/regain.431efde9.js",
         "httpversion" => "1.1",
    "http_status_code" => "304",
               "bytes" => "0",
        "http_referer" => "https://uatest.winfae.com/wechat/regain.html"
		
\S+ 和 [^\n\t\r\f]+ 语法一样   非空格

		

 
 my $str="  begin 123.456 end  ";  
if ($str =~/(?<request_time>\d+\.\d+)/)  
   {  
    my ($request_time) = ($+{request_time});    
   print $request_time."\n";};  
zjtest7-frontend:/root/0825# perl a1.pl   
123.456  

 "http_referer" => "https://uatest.winfae.com/wechat/regain.html"
 
  \"(?<http_referer>\S+)\"
  
  
 \"(?<http_user_agent>\S+)\"	
 \"Mozilla/5.0 (Linux; Android 5.1.1; vivo X6S A Build/LMY47V) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/37.0.0.0 Mobile MQQBrowser/6.2 TBS/036558 Safari/537.36 MicroMessenger/6.3.25.861 NetType/WIFI Language/zh_CN\"
 
 
              "message" => " 121.40.205.143 [29/Aug/2016:13:54:08 +0800] \"GET /resources/plugins/artDialog/ui-dialog.css HTTP/1.1\"200 9985 \"https://uatest.winfae.com/wechat/home.html?useragent=ios_h5_zjcap&apiver=2&WKWebView=1\" \"ios_h5_zjcap\"",
            "@version" => "1",
          "@timestamp" => "2016-08-29T05:56:53.217Z",
                "path" => "/rsyslog/data/nginx/uat/nginx_access01_log.2016-08-29",
                "host" => "0.0.0.0",
                "type" => "uat_nginx_access",
            "clientip" => "121.40.205.143",
                "time" => "29/Aug/2016:13:54:08 +0800",
                "verb" => "GET",
             "request" => "/resources/plugins/artDialog/ui-dialog.css",
         "httpversion" => "1.1",
    "http_status_code" => "200",
               "bytes" => "9985",
        "http_referer" => "https://uatest.winfae.com/wechat/home.html?useragent=ios_h5_zjcap&apiver=2&WKWebView=1",
     "http_user_agent" => "ios_h5_zjcap"
}

{
       "message" => " 121.40.205.143 [29/Aug/2016:13:59:35 +0800] \"GET /resources/js/toolbar.49fc367e.js?_v=${last.updated}&_=1472450673142 HTTP/1.1\"200 1800 \"https://uatest.winfae.com/products/productList.html\" \"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.87 Safari/537.36\"",
      "@version" => "1",
    "@timestamp" => "2016-08-29T06:02:18.775Z",
          "path" => "/rsyslog/data/nginx/uat/nginx_access01_log.2016-08-29",
          "host" => "0.0.0.0",
          "type" => "uat_nginx_access",
          "tags" => [
        [0] "_grokparsefailure"
    ]
}
 
 

ELK错误1_Kafka->Logstash->Elasticsearch过程,Elasticsearch报grokparsefailure错误

实验背景: Kafka->Logstash->Elasticsearch实验环境安装完成; Logstash的启动文件设置有过滤机制; Logstash启动文件: input { kafka {   ...

logstash使用grok正则解析日志

http://xiaorui.cc/2015/01/27/logstash%E4%BD%BF%E7%94%A8grok%E6%AD%A3%E5%88%99%E8%A7%A3%E6%9E%90%E6%9...
  • earbao
  • earbao
  • 2015年10月21日 16:38
  • 28888

logstash 学习小记

简单的介绍了logstash,给出一些简单实用的examples

Logstash 配置总结

#整个配置文件分为三部分:input,filter,output #参考这里的介绍 https://www.elastic.co/guide/en/logstash/current/configur...
  • xfg0218
  • xfg0218
  • 2016年10月31日 11:02
  • 4512

初识Logstash

logstash简介logstash是一种分布式日志收集框架,开发语言是JRuby,当然是为了与Java平台对接,不过与Ruby语法兼容良好,非常简洁强大,经常与ElasticSearch,Kiban...

Logstash使用grok解析IIS日志

Logstash使用grok解析IIS日志 1. 安装配置 安装Logstash前请确认Elasticsearch已经安装正确,参见RedHat6.4安装Elasticsearch5.2.0。 ...

logstash使用grok正则解析日志和kibana遇到的问题

#Nginx日志格式定义 log_format combine '$remote_addr - $remote_user [$time_local] "$request" $http_host ...

Logstash的数据类型和基本语法

From:http://www.ttlsa.com/elk/elk-logstash-configuration-syntax/ logstash支持的数据类型有: array 数组...

Logstash 常用正则(grok-patterns)

USERNAME [a-zA-Z0-9._-]+ USER %{USERNAME} INT (?:[+-]?(?:[0-9]+)) BASE10NUM (?[+-]?(?:(?:[0-9]+(?:\....

logstash + grok 正则语法

logstash + grok 正则语法
  • signmem
  • signmem
  • 2017年02月28日 17:58
  • 685
内容举报
返回顶部
收藏助手
不良信息举报
您举报文章:logstash grok正则调试
举报原因:
原因补充:

(最多只允许输入30个字)