logstash 嵌套json/多层json解析配置

最近解析日志时，发现日志最多有六层嵌套json，且每层json的位置也经常变换，导致正则解析经常失效，并且最终解析文件过于庞大；

最终改成json方式解析日志。

日志样例：

<150>Mar 10 13:22:57 localhost sdp-proxy@userLog[486]: {"event":{"subType":"user.l3app.access","mainType":"app","timestamp":"2022-03-10 13:22:57","level":"INFO","reason":"","result":"SUCCESS"},"actor":{"id":"12de118d-908b-11ec-8984-27a2f4932f98","domain":"local","type":"user","email":"s****i@126.com","groupPath":"/分支机构/营业部/营业部","phone":"15******805","label":"","detail":"","username":"shany"},"traceId":"1de9c96442cd65d7-17411437948","target":{"id":"fd8c80d0-7aac-11ec-9784-29e69891c97a","details":{"app":{"status":{"responseTime":114,"upstreamConnectTime":0,"upstreamHeaderTime":115,"upstreamResponseTime":115,"sendResponseTime":0,"recvBytes":588,"sendBytes":1793},"web":{"request":{"refer":"http://172.16.32.137/apps/frame/pages/genericParam.html?VIEW_ID=YGT_MY_BUSI_TRACK&noscroll=false","url":"http://172.20.32.137:80/kjdp_ajax","body":"","method":"POST","query":"services=YG003785&returnType=json","xForwardedFor":"","requestLine":"POST /kjdp_ajax?services=YG003785&returnType=json HTTP/1.1","backendUrl":"http://172.16.32.137:80/kjdp_ajax"},"response":{"contentLength":"","contentDisposition":"","attachment":"false","status":"200","redirectUri":"","server":"Tengine/2.1.0","contentType":"text/json;charset=UTF-8"}},"upstream":{"host":"172.16.32.137","port":80}}},"type":"l3app","displayName":"业务生产_综合运营平台_2.137_WEB"},"source":{"id":"E54D9B75","type":"proxy","displayName":"aTrust"},"client":{"deviceId":"0D8EEB802D07566DDC16187982EE03EC","port":"2488","ip":"60.22.24.30","virtualIp":"","mac":"70:B5:E8:22:3F:0D","userAgent":{"rawUserAgent":"Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.39 Safari/537.36","processName":"","browser":"Chrome31.0.1650.39","processPath":"","os":"Windows 8.1"},"device":"SDPBrowserClient"}}

工具解析json预览：

据统计：字段大约62个，需要提取字段12个

配置文件：

logstash.conf 

if [hostIP] == "127.0.0.1"  {

         mutate {

                replace => ["product","零"]

                replace => ["vendor","服"]

         }

       grok {

           match => [

"raw_event",'<\d+>\w+\s+\S+\s+\S+\s+\S+\s\S+\s(?<temp>.*)'

          ]

        }

json {

        source => "temp"

        remove_field => [ "temp"]

        }



 mutate {

   add_field => ["src_port","%{[client][port]}","src_ip","%{[client][ip]}","src_mac","%{[client][mac]}"]

   add_field => ["dest_ip","%{[target][details][app][upstream][host]}","dest_port","%{[target][details][app][upstream][port]}"]

   rename => ["[event][result]","result"]

   rename => ["[event][subType]","name"]

   rename => ["[actor][username]","username"]

   rename => ["[actor][domain]","dvc_domain"]

   rename => ["[actor][groupPath]","area"]

   rename => ["[target][details][app][web][response][status]","status"]

   rename => ["[event][level]","level"]

   rename => ["[target][displayName]","dest_service_name"]

   rename => ["[target][details][app][web][request][url]","url"]

   rename => ["[target][details][app][web][request][method]","http_method"]

}



 mutate {

   remove_field => ["traceId","[target]","[client]","[actor]","[event]","[source]"]

}



if [src_ip] !~ "[0-9.]+" {

   mutate {

        remove_field => ["src_ip"]

        }

        }

if [src_port] !~ "[0-9]+" {

   mutate {

        remove_field => ["src_port"]

        }

        }

if [dest_ip] !~ "[0-9.]+" {

   mutate {

        remove_field => ["dest_ip"]

        }

        }

if [dest_port] !~ "[0-9]+" {

   mutate {

        remove_field => ["dest_port"]

        }

        }



   }

配置解决问题：

1、解决字段空值（包括IP和端口）

2、字段内容变换问题

配置思路：

1、json文件整体解析

2、主字段改名、重新赋值字段

3、删除多余的字段（删除顶层嵌套，前提是主要字段已rename或者重新赋值到其他字段）

4、IP、端口空值判断