logstash7.8 apache日志解析 grok

1. apache日志

27.60.18.21 - - [21/Jul/2020:10:10:10 +0530] "GET /api/v1.2/places/search/json?username=liusg&location=28.5359586,77.3677936&query=elasticsearch&explain=true&bridge=true HTTP/1.1" 200 3284
27.60.18.22 - - [22/Jul/2020:10:11:11 +0530] "POST /api/v1.0/places/search/json?username=liushangguo&location=28.5359586,77.3677936&query=docker&explain=true&bridge=true HTTP/1.1" 200 1452
27.60.18.23 - - [23/Jul/2020:12:12:12 +0530] "HEAD /api/v1.2/places/nearby/json?&refLocation=28.5359586,77.3677936&keyword=FINATM HTTP/1.1" 200 3283
27.60.18.24 - - [24/Jul/2020:13:13:13 +0530] "POST /api/v2.0/places/search/json?username=liu.sg&location=28.5359586,77.3677936&query=iphone&explain=true&bridge=true HTTP/1.1" 200 3415
27.60.18.25 - - [25/Jul/2020:16:16:16 +0530] "GET /api/v1.2/places/search/json?username=pradeep.pgu&location=28.5359586,77.3677936&query=15000227329&explain=true&bridge HTTP/1.1" 200 2476

2. logstash配置

input {
    file {
        path => "/var/log/aaa.log"
        stat_interval => 1
        start_position => "beginning"
    }
}

filter {
    grok {
        match => { 
            "message" => "%{IPORHOST:client_ip} %{HTTPDUSER:ident} %{HTTPDUSER:auth} \[%{HTTPDATE:req_time}\] \"(?:%{WORD:method} /api/v%{NUMBER:api_version}/.*/json\?%{NOTSPACE:req_data}(?: HTTP/%{NUMBER:http_version})?|%{DATA:rawrequest})\" %{NUMBER:resp_code} (?:%{NUMBER:req_byte}|-)"
        }
    }
	
    kv { 
        source => "req_data"
        field_split => "&"
    }

    if [query] {
        mutate {
            add_field => { "search" => "%{query}" }
        }
    } else if [keyword] {
        mutate {
            add_field => { "search" => "%{keyword}" }
        }
    }

    if [refLocation] {
        mutate {
            rename => { "refLocation" => "location" }
        }
    }

#  新增timestamp字段，将@timestamp时间增加8小时
   ruby { code => "event.set('timestamp', event.get('@timestamp').time.localtime + 8*60*60)" }
   ruby { code => "event.set('@timestamp',event.get('timestamp'))" }
		
	mutate{
		split=>["location",","]
		add_field => { "jing" => "%{[location][0]}" }
		add_field => { "wei" => "%{[location][1]}"	}
	}
 
   mutate {
		# 删除指定字段,message
	    remove_field => ["@version","host","path","tags"]
		# 如果将@timestamp字段删除，自动生成索引的日期配置就为空		
		# remove_field => ["@timestamp"]
   }
}

output {
	stdout {}
	elasticsearch {
		hosts => ["http://192.168.1.58:9200"]
		index => "httpd-%{+YYYY.MM.dd}"
		user => "elastic"
		password => "123456"
	}
}

3. 追加日志

echo '20.10.20.22 - - [25/Aug/2010:20:20:20 +0530] "GET /api/v2.2/places/search/json?username=sdd.pgu&location=28.2222233,77.3333333&query=elasticssss&explain=true&bridge HTTP/1.1" 200 2223' >> /var/log/aaa.log

4. 索引查询结果

{
  "took" : 4,
  "timed_out" : false,
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "skipped" : 0,
    "failed" : 0
  },
  "hits" : {
    "total" : {
      "value" : 1,
      "relation" : "eq"
    },
    "max_score" : 1.0,
    "hits" : [
      {
        "_index" : "httpd-2020.07.23",
        "_type" : "_doc",
        "_id" : "S415enMB5upHvde6CV8J",
        "_score" : 1.0,
        "_source" : {
          "auth" : "-",
          "api_version" : "2.2",
          "ident" : "-",
          "method" : "GET",
          "location" : [
            "28.2222233",
            "77.3333333"
          ],
          "@timestamp" : "2020-07-23T15:00:19.003Z",
          "req_time" : "25/Aug/2010:20:20:20 +0530",
          "username" : "sdd.pgu",
          "query" : "elasticssss",
          "message" : """20.10.20.22 - - [25/Aug/2010:20:20:20 +0530] "GET /api/v2.2/places/search/json?username=sdd.pgu&location=28.2222233,77.3333333&query=elasticssss&explain=true&bridge HTTP/1.1" 200 2223""",
          "wei" : "77.3333333",
          "explain" : "true",
          "req_byte" : "2223",
          "search" : "elasticssss",
          "http_version" : "1.1",
          "client_ip" : "20.10.20.22",
          "timestamp" : "2020-07-23T15:00:19.003Z",
          "resp_code" : "200",
          "req_data" : "username=sdd.pgu&location=28.2222233,77.3333333&query=elasticssss&explain=true&bridge",
          "jing" : "28.2222233"
        }
      }
    ]
  }
}

5. grok表达式

6. grok表达式-案例

6.1. 日志

Jan 1 06:25:43 mailserver14 postfix/cleanup[21403]: BEF25A72965: message-id=<20130101142543.5828399CCAF@mailserver14.example.com> Alert

6.2. grok表达式

%{MONTH:month} %{NUMBER:day} %{TIME:time} %{WORD:ms} (?:%{WORD:postfix}/%{WORD:status}\[%{NUMBER:bbyte}\]: %{WORD:ms22}: (?<sssd>.*>)) %{LOGLEVEL:level}

6.3. 拆分结果

{
  "level": "Alert",
  "ms": "mailserver14",
  "sssd": "message-id=<20130101142543.5828399CCAF@mailserver14.example.com>",
  "month": "Jan",
  "ms22": "BEF25A72965",
  "bbyte": "21403",
  "time": "06:25:43",
  "postfix": "postfix",
  "day": "1",
  "status": "cleanup"
}