logstash remove field message

               

{             "message" => "10.10.17.1 [11/Jan/2017:13:21:23 +0800] \"GET /resources/js/toolbar.js?_=1484112094581 HTTP/1.1\" - 200 2775 \"http://10.10.17.2/\" \"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:50.0) Gecko/20100101 Firefox/50.0\" 0.000 -",            "@version" => "1",          "@timestamp" => "2017-01-11T05:21:23.000Z",                "path" => "/var/log/nginx/access.log",                "host" => "db01",                "type" => "nginx_access",            "clientip" => "10.10.17.1",                "time" => "11/Jan/2017:13:21:23 +0800",                "verb" => "GET",             "request" => "/resources/js/toolbar.js",         "httpversion" => "1.1",    "http_status_code" => "200",               "bytes" => "2775",        "http_referer" => "http://10.10.17.2/",     "http_user_agent" => "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:50.0) Gecko/20100101 Firefox/50.0",       "response_time" => 0.0,            "messager" => "nginx_access-10.10.17.1 [11/Jan/2017:13:21:23 +0800] \"GET /resources/js/toolbar.js?_=1484112094581 HTTP/1.1\" - 200 2775 \"http://10.10.17.2/\" \"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:50.0) Gecko/20100101 Firefox/50.0\" 0.000 -"}{            "@version" => "1",          "@timestamp" => "2017-01-11T06:06:09.000Z",                "path" => "/var/log/nginx/access.log",                "host" => "db01",                "type" => "nginx_access",            "clientip" => "10.10.17.1",                "time" => "11/Jan/2017:14:06:09 +0800",                "verb" => "GET",             "request" => "/resources/images/home/QR_code.jpg",         "httpversion" => "1.1",    "http_status_code" => "200",               "bytes" => "52810",        "http_referer" => "http://10.10.17.2/",     "http_user_agent" => "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:50.0) Gecko/20100101 Firefox/50.0",       "response_time" => 0.0,            "messager" => "nginx_access-10.10.17.1 [11/Jan/2017:14:06:09 +0800] \"GET /resources/images/home/QR_code.jpg HTTP/1.1\" - 200 52810 \"http://10.10.17.2/\" \"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:50.0) Gecko/20100101 Firefox/50.0\" 0.000 -"}    }                   mutate {                        convert => [ "request_time", "float"]                        add_field =>["response_time","%{request_time}"]                        convert => [ "response_time", "float"]                        add_field => [ "[@metadata][zabbix_key]" , "logstash-api-access" ]                        add_field => [ "[@metadata][zabbix_host]" , "dr-mysql01" ]                        add_field =>["messager","%{type}-%{message}"]                        remove_field =>["request_time"]                        remove_field =>["message"][elk@db01 nginx]$ cat logstash_nginx.conf input {           file {                 type => "wj_frontend_access"                 path => ["/data01/applog_backup/winfae_log/wj-frontend0*access*"]         }          file {                type => "nginx_access"                path => ["/var/log/nginx/access.log"]        } }filter {    grok {        match =>[              "message","%{IPORHOST:clientip} \[%{HTTPDATE:time}\] \"%{WORD:verb} %{URIPATHPARAM:request}\?.* HTTP/%{NUMBER:httpversion}\" \- %{NUMBER:http_status_code} %{NUMBER:bytes} \"(?<http_referer>\S+)\" \"(?<http_user_agent>(\S+\s+)*\S+)\" (%{BASE16FLOAT:request_time}) (%{IPORHOST:http_x_forwarded_for}|-)",              "message" , "%{IPORHOST:clientip} \[%{HTTPDATE:time}\] \"%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}\" \- %{NUMBER:http_status_code} %{NUMBER:bytes} \"(?<http_referer>\S+)\" \"(?<http_user_agent>(\S+\s+)*\S+)\" (%{BASE16FLOAT:request_time}) (%{IPORHOST:http_x_forwarded_for}|-)",             "message","%{IPORHOST:clientip} \[%{HTTPDATE:time}\] \"%{WORD:verb} (?<http_url>\S+)\s+HTTP/%{NUMBER:httpversion}\"\s+\-\s+%{NUMBER:http_status_code}\s+%{NUMBER:bytes}\s+\"\-\"\s+\"(?<http_user_agent>(\S+))\"\s+(%{BASE16FLOAT:request_time})\s+(%{IPORHOST:http_x_forwarded_for}|-)",             "message","%{IPORHOST:clientip} \[%{HTTPDATE:time}\] \"%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}\" \- %{NUMBER:http_status_code} %{NUMBER:bytes} \"\" \"(?<http_user_agent>(\S+\s+)*\S+)\" (%{BASE16FLOAT:request_time}) (%{IPORHOST:http_x_forwarded_for}|-)"                     ]    }                   mutate {                        convert => [ "request_time", "float"]                        add_field =>["response_time","%{request_time}"]                        convert => [ "response_time", "float"]                        add_field => [ "[@metadata][zabbix_key]" , "logstash-api-access" ]                        add_field => [ "[@metadata][zabbix_host]" , "dr-mysql01" ]                        add_field =>["messager","%{type}-%{message}"]                        remove_field =>["request_time"]                        remove_field =>["message"]                       # remove_field =>["messager"]                }              date {        match => ["time", "dd/MMM/yyyy:HH:mm:ss Z"]    }     }output {        stdout {                        codec => rubydebug                } #        if [response_time] >= 5  {#          zabbix {#                zabbix_host => "[@metadata][zabbix_host]"#                zabbix_key => "[@metadata][zabbix_key]"#        zabbix_server_host => "192.168.32.55"#        zabbix_server_port => "10051"#                zabbix_value => "messager"#        }#          }     if [type] == "nginx_access" {         redis {                host => "127.0.0.1"                data_type => "list"                key => "nginx_access:redis"                port=>"6379"                password => "1234567"        }}      else if [type] == "wj_frontend_access"{       redis {                 host => "127.0.0.1"                 data_type => "list"                 key => "wj_frontend_access:redis"                 port=>"6379"                 password => "1234567"         } }}如果你把 "message" 里所有的信息都 grok 到不同的字段了，数据实质上就相当于是重复存储了两份。所以你可以用 remove_field 参数来删除掉 message 字段，或者用 overwrite 参数来重写默认的 message 字段，只保留最重要的部分。重写参数的示例如下：filter {    grok {        patterns_dir => "/path/to/your/own/patterns"        match => {            "message" => "%{SYSLOGBASE} %{DATA:message}"        }        overwrite => ["message"]    }}

           

再分享一下我老师大神的人工智能教程吧。零基础！通俗易懂！风趣幽默！还带黄段子！希望你也加入到我们人工智能的队伍中来！https://blog.csdn.net/jiangjunshow