logstash grok mysql_Logstash--grok 正则表达式

我在使用logstash的时候，为了更细致的切割日志，会写一些正则表达式。

使用方法input {

file {

type => "billin"

path => "/data/logs/product/result.log"

}

}

filter {

grok {

type => "billin"

pattern => "%{BILLINCENTER}"

patterns_dir => "/data/logstash/patterns/my_patterns"

}

}

output {

redis {

host => "192.168.50.13"

data_type =>"list"

key => "logstash:redis"

}

}

以下内容为正则表达式文件：cat my_patterns

TAB \t

META \-+

WZ ([^ ]*)

IPPORT %{IP}:%{POSINT}|%{META}

REQUEST (?:/[A-Za-z0-9$.+!*'(),~:#%_-]*)+\?[A-Za-z0-9$.+!*'(),~#%&/=:;_-]*

TY (?:(?

#EVERYURL ((\w+://)?([^\.]+)(\.[^/:]+)(:\d*)?([^#]*))|-

#EVERYURL (((\w+://)?([^\.]+)(\.[^/:]+)?([^#]*))+)|(\w+)|-

#EVERYURL ((\w+://)?([^\.]+)(\.[^/:]+)?([^#]*))+)|-

EVERYURL (http://+[\w\d:#@%/;$()~_?\+-=\\\.&]+)|(-)

#Logformat

########nginx access log example########

#122.137.199.113"122.137.199.113"www.xxxx.com172.16.10.110172.16.12.114:8018/Jun/2013:15:51:03 +0800GET /g/getSaleCounts.do?rnd=1371541857448&showStatus=true&goodsIds=215abd2e8fa95bc8 HTTP/1.120078"http://www.xxxx.com/goods-215abd2e8fa95bc8.html""Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; NP06)""a8fdb711-a695-43bd-abdd-a224fb07350d"

###############################

NGINXACCESSLOG %{IP:remote_ip}%{SPACE}%{QS:x_forward}%{SPACE}%{HOSTNAME:server_name}%{SPACE}%{IP:server_ip}%{SPACE}%{IPPORT:upstrem_ip}%{SPACE}%{HTTPDATE:timestamp}%{SPACE}%{WORD:verb}%{SPACE}%{WZ:request}%{SPACE}HTTP/%{NUMBER:httpversion}%{SPACE}%{NUMBER:response}%{SPACE}%{NUMBER:bytes}%{SPACE}%{QS:uri}%{SPACE}%{QS:agent}%{SPACE}%{QS:guid}

#picture p0.xxxx.com access log . 2012.07.19 add

PICLOG %{IP:remote_ip}%{SPACE}%{QS:x_forward}%{SPACE}%{HOSTNAME:server_name}%{SPACE}%{IP:server_ip}%{SPACE}%{HTTPDATE:timestamp}%{SPACE}%{WORD:verb}%{SPACE}%{WZ:request}%{SPACE}HTTP/%{NUMBER:httpversion}%{SPACE}%{NUMBER:response}%{SPACE}%{NUMBER:bytes}%{SPACE}%{QS:uri}%{SPACE}%{QS:agent}

#iis log format 20120618 add

###########iis log example###############

#2013-06-18 08:00:00 172.16.10.233 GET /js/functions.js - 80 - 117.136.34.2 Mozilla/5.0+(Linux;+U;+Android+4.1.2;+zh-CN;+LT22i+Build/6.2.A.0.400)+AppleWebKit/534.31+(KHTML,+like+Gecko)+UCBrowser/9.0.1.275+U3/0.8.0+Mobile+Safari/534.31 200 0 0 0

###################################

IISLOG %{DATE_EU:log_date} %{TIME:log_time} %{IP:server_ip} %{WORD:verb} %{URIPATH:uri_stem} %{WZ:uri_query} %{POSINT:s_port} %{WZ:cs_username} %{IP:c_ip} %{WZ:agent} %{POSINT:request} %{POSINT:substatus} %{POSINT:win32_status} %{POSINT:time_taken}

#2012/07/12 add

ZW \w+

###java date example

# 2012-11-27 14:52:42

############

JAVA_DATE %{DATE_EU} %{TIME}

EARTHLOG \[%{JAVA_DATE:log_date}\] \[%{WORD:level}\] \[%{WORD:action}\] \[\{"desc":"%{ZW:desc}","dateTime":%{ZW:dateTime},"userId":"%{ZW:userId}","code":%{ZW:code}\}\]

EAGLEUPDATE \[%{JAVA_DATE:log_date}\] \[%{WORD:level}\] \[%{WORD:action}\] \[\{"desc":%{QS:desc},"dateTime":%{ZW:dateTime},"userId":"%{ZW:userId}","code":%{ZW:code},"orderId":"%{ZW:orderId}"\}\]

EAGLELOGIN \[%{JAVA_DATE:log_date}\] \[%{WORD:level}\] \[%{WORD:action}\] \[\{"desc":%{QS:desc},"dateTime":%{ZW:dateTime},"userId":"%{ZW:userId}","code":%{ZW:code}\}\]

#2012/10/23 add

LJF (-\s+-)

RESINLOG %{IP:remote_ip}%{SPACE}%{NUMBER}%{SPACE}%{LJF}%{SPACE}\[%{HTTPDATE:timestamp}\]%{SPACE}"%{WORD:verb}%{SPACE}%{WZ:request}%{SPACE}HTTP/%{NUMBER}"%{SPACE}%{NUMBER:response}%{SPACE}%{NUMBER:bytes}%{SPACE}%{QS:uri}%{SPACE}%{QS:agent}%{SPACE}%{QS:session}

#RESINLOG %{IP:ip} %{NUMBER} - - \[%{HTTPDATE:time}\] "%{WORD:verb} %{WZ:request} HTTP/%{NUMBER}" %{NUMVER:response} %{NUMBER:bytes} %{QS:uri} %{QS:agent} %{QS:session}

#2012/11/13 add

DKH (\{.*\})

STOREGREP (\[\/\/\/ \- \] INFO \-)

DHMH ([^;|=]*)

CENTERLOG %{JAVA_DATE} %{STOREGREP} BId=%{NUMBER:bid};BR=%{DHMH:br};BP=%{DKH:bp}

#2012/11/20 add

JAVAGREP (\[\/\/\/ \- \])

ORDERCENTERERR %{JAVA_DATE} \[ RMI TCP Connection\(%{NUMER:threadid}\) -%{IP:ip}\] %{JAVAGREP} %{WORD:level}%{SPACE}%{WZ} - %{QS:message}

ORDERCENTERRESULT %{JAVA_DATE} \[ RMI TCP Connection\(%{NUMER:threadid}\) -%{IP:ip}\] %{JAVAGREP} %{WORD:level}%{SPACE}%{WZ} - %{DKH:message}

#2012/11/27 add

#####log example#######

#2013-06-18 15:28:12 INFO :{message:媒体传递的参数{"uid":["0"],"cid":["A100054947||0000"],"url":["http://www.xxxx.com/?from=lianmeng-weiyi"],"src":["weiyi"]}}

#

PARTNER %{JAVA_DATE:timestamp} %{WORD:level} :%{DKH:message}

#2012/11/28 add

PARTNERAPI %{JAVA_DATE:timestamp} %{WZ:level} :%{DKH:message}

#2013/06/18 add

#pattern all in the '[adskfjl }{\]'

FKH ([^;]*)

#######aether.log#####

#[2013-06-18 15:27:29] [INFO] [com.tuan.web.controller.IndexController] [{message:setHotStore#hot store size:5}]

AETHERLOG \[%{JAVA_DATE:timestamp}\] \[%{WZ:level}\] \[%{WZ:method}\] %{FKH:message}