logstash配置文件，grok过滤，geoip地理

Mess_Me

已于 2025-04-30 15:30:47 修改

阅读量983

点赞数

分类专栏： Linux 文章标签： linux

于 2021-03-04 16:20:43 首次发布

本文链接：https://blog.csdn.net/weixin_44221035/article/details/114369495

版权

Linux 专栏收录该内容

20 篇文章

订阅专栏

logstash配置文件。

input {
    beats {
        port => 5044
    }
}
input {
    syslog {
        port => 514
    }
}

filter{
	#去除#号开头
    if ([message] =~ "^#") {
        drop {}
    }
    #匹配filetype值为Cache开头。
    if ([filetype] =~ "^Cache") {
    	#进行grok正则匹配message字段值，下文会有正则
        grok {
            patterns_dir => "/etc/logstash/nginx/nginx"
            match => { "message" => "%{NGINXACCESS}" }
            remove_field => ["message"]
        }
        #匹配非私网ip地址
        if [ClientIP] !~ "^127\.|^192\.168\.|^172\.1[6-9]\.|^172\.2[0-9]\.|^172\.3[01]\.|^10\." {
        	#通过geoip库获取ip的地理信息，从而进行kiban——map
            geoip {
                source => "ClientIP" #ip字段
                target => "geoip"
                database => "/etc/logstash/db/GeoLite2-City.mmdb" #数据库存放路径
                #只留取需要字段，在新版ekl中不需要去创建多余字段，获取经纬数据直接使用location字段即可
                fields => ["city_name","continent_code","country_code2","country_code3","country_name","dma_code","ip","location","postal_code","region_name","timezone"]
            }
        }
        #删除grok匹配后多余的字段，主要是filebeat收集后会添加许多多余字段
        mutate{
            remove_field => ["log"]
            remove_field => ["host"]
            remove_field => ["architecture"]
            remove_field => ["ecs"]
            remove_field => ["agent"]
            remove_field => ["http_referrer"]
            remove_field => ["tags"]
            remove_field => ["input"]
            remove_field => ["httpagent"]
        }
    }
    if ([filetype] =~ "^Fastdfs_Tracker") {
        grok {
            patterns_dir => "/etc/logstash/nginx/nginx"
            match => { "message" => "%{FASTDFSNGINX}" }
            remove_field => ["message"]
         }
        mutate{
            remove_field => ["auth"]
            remove_field => ["tags"]
            remove_field => ["agent"]
            remove_field => ["ecs"]
            remove_field => ["log"]
            remove_field => ["input"]
            remove_field => ["host"]
        }

    }
    if ([filetype] =~ "^Fastdfs_Storage") {
        grok {
            patterns_dir => "/etc/logstash/nginx/nginx"
            match => { "message" => "%{STORAGENGINX}" }
            remove_field => ["message"]
         }
        mutate{
            remove_field => ["auth"]
            remove_field => ["tags"]
            remove_field => ["agent"]
            remove_field => ["ecs"]
            remove_field => ["log"]
            remove_field => ["input"]
            remove_field => ["host"]
        }
    }
}

output {
    if ([filetype] =~ "^Cache") {
        elasticsearch{
            index => "cache-%{+YYYY.MM}" #创建index，每月创建一个
            hosts => ["host_ip:9200"]
            user => "elastic" #没有做验证这个不需要
            password => "password"
        }
    }
    if ([filetype] =~ "^Fastdfs_Tracker") {
        elasticsearch{
            index => "tracker-%{+YYYY.MM}"
            hosts => ["host_ip:9200"]
            user => "elastic"
            password => "password"
        }
    }
    if ([filetype] =~ "^Fastdfs_Storage") {
        elasticsearch{
            index => "storage-%{+YYYY.MM}"
            hosts => ["host_ip:9200"]
            user => "elastic"
            password => "password"
        }
    }
    if ([host] =~ "x.x.x.x") {   #匹配syslog日志
        elasticsearch{
            index => "syslog-%{+YYYY.MM}"
            hosts => ["host_ip:9200"]
            user => "elastic"
            password => "password"
        }
    }
}

logstash添加ca证书并用jdbc读取数据库信息和mail简单报警操作

input {
    syslog {
        port => 514
    }
    jdbc {
        jdbc_connection_string => "jdbc:mysql://x.x.x.x:3306/sitex3"
        jdbc_user => "usr_sitex3"
        jdbc_password => "passwd"
        jdbc_validate_connection => true
        jdbc_driver_library => "/etc/logstash/mysql/mysql-connector-java-8.0.23.jar"
        jdbc_driver_class => "com.mysql.jdbc.Driver"
        jdbc_paging_enabled => "true"
        jdbc_page_size => "50000"
        schedule => "*/1 * * * *"
        statement => "SELECT * from t_login_logout_events where operate_time > :sql_last_value"
        type => "user_login"  #添加字段
    }
}


filter {
    if [host] == "x.x.x.x" {
        grok {
            patterns_dir => "/etc/logstash/syslog/ips"
            match => { "message" => "%{IPS}" }
        }
        if [status] == "255" {
            grok {
                patterns_dir => "/etc/logstash/syslog/ips"
                match => { "message" => "%{LOGIN}" }
            }
        }
    }
}

output{
    if [host] == "x.x.x.x" {
    #邮件告警配置，匹配某个字段进行报警，也可以做正则进行内容匹配
#        email {
#            port => 25
#            address => "mail.xxxx.com.cn"
#            username => "xxxxxxxx@xxxx.com.cn"
#            password => "xxxxxx"
#            authentication => "plain"
#            use_tls => false
#            from => "xxxxxxxx@xxxx.com.cn"
#            subject  => "Warning: waf"
#            to  => "xxxxxxxx@xxxx.com.cn"
#            via => "smtp"
#            body => "告警信息：%{message}"
#        }
        elasticsearch{
            cacert => "/etc/logstash/ca/client-ca.cer"
            ssl => true
            ssl_certificate_verification => false
            index => "waf-%{+YYYY.MM}"
            hosts => ["https://x.x.x.x:9200"]
            user => "elastic"
            password => "passwd"
        }
    }
# 数据哭登录日志
    if [type] == "xiniu_login" {
        elasticsearch{
            cacert => "/etc/logstash/ca/client-ca.cer"
            ssl => true
            ssl_certificate_verification => false
            index => "xiniu-login-%{+YYYY.MM}"
            hosts => ["https://x.x.x.x:9200"]
            user => "elastic"
            password => "passwd"
        }
    }

Grok配置文件

/etc/logstash/nginx/nginx文件配置

NGINXACCESS %{IP:ClientIP}\| %{HTTPDUSER:ident}\|%{HOSTNAME:domain}\| %{WORD:request_method} %{NOTSPACE:request} HTTP/%{NUMBER:httpversion}\|  %{NUMBER:status:int}\| \"%{GREEDYDATA:http_referrer}\"\|  \"%{GREEDYDATA:httpagent}\"\|%{NUMBER:body_sent:int}\| %{NUMBER:request_time:float}\| (%{NUMBER:response_time:float}|-)
FASTDFSNGINX %{IP:clientip} - %{USER:auth} \[%{HTTPDATE:timestamp}\] \"%{WORD:request_method} %{NOTSPACE:request} HTTP/%{NUMBER:httpversion}\" %{NUMBER:status:int} %{NUMBER:sc_bytes:int} \"(?:%{URI:referrer}|-)\" \"%{GREEDYDATA:agent}\"
STORAGENGINX %{IP:clientip} - %{USER:auth} \[%{HTTPDATE:timestamp}\] \"%{WORD:request_method} %{NOTSPACE:request} HTTP/%{NUMBER:httpversion}\" %{NUMBER:status:int} %{NUMBER:sc_bytes:int} \"(?:%{URI:referrer}|-)\" \"%{GREEDYDATA:agent}\"
FTP \(%{NUMBER:id}\) %{DATE_CH:date} %{TIME:time} - (?<type>\(not logged in\)|ftp_log) \(%{IP:ip}\)\> %{GREEDYDATA:data}

以下是一位大神的nginx正则匹配的对照表，我使用的是kibana自带grok进行调试原文连接https://www.cnblogs.com/lize3379/p/6118788.html
在这里插入图片描述

Grok预定义正则

vim /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-patterns-core-4.1.2/patterns/grok-patterns
USERNAME [a-zA-Z0-9._-]+
USER %{USERNAME}
EMAILLOCALPART [a-zA-Z][a-zA-Z0-9_.+-=:]+
EMAILADDRESS %{EMAILLOCALPART}@%{HOSTNAME}
INT (?:[+-]?(?:[0-9]+))
BASE10NUM (?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+)))
NUMBER (?:%{BASE10NUM})
BASE16NUM (?<![0-9A-Fa-f])(?:[+-]?(?:0x)?(?:[0-9A-Fa-f]+))
BASE16FLOAT \b(?<![0-9A-Fa-f.])(?:[+-]?(?:0x)?(?:(?:[0-9A-Fa-f]+(?:\.[0-9A-Fa-f]*)?)|(?:\.[0-9A-Fa-f]+)))\b

POSINT \b(?:[1-9][0-9]*)\b
NONNEGINT \b(?:[0-9]+)\b
WORD \b\w+\b
NOTSPACE \S+
SPACE \s*
DATA .*?
GREEDYDATA .*
QUOTEDSTRING (?>(?<!\\)(?>"(?>\\.|[^\\"]+)+"|""|(?>'(?>\\.|[^\\']+)+')|''|(?>`(?>\\.|[^\\`]+)+`)|``))
UUID [A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12}
# URN, allowing use of RFC 2141 section 2.3 reserved characters
MAC (?:%{CISCOMAC}|%{WINDOWSMAC}|%{COMMONMAC})
CISCOMAC (?:(?:[A-Fa-f0-9]{4}\.){2}[A-Fa-f0-9]{4})
WINDOWSMAC (?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2})
COMMONMAC (?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2})
IP (?:%{IPV6}|%{IPV4})
HOSTNAME \b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\.?|\b)
IPORHOST (?:%{IP}|%{HOSTNAME})
HOSTPORT %{IPORHOST}:%{POSINT}

# paths
PATH (?:%{UNIXPATH}|%{WINPATH})
UNIXPATH (/([\w_%!$@:.,+~-]+|\\.)*)+
TTY (?:/dev/(pts|tty([pq])?)(\w+)?/?(?:[0-9]+))
WINPATH (?>[A-Za-z]+:|\\)(?:\\[^\\?*]*)+
URIPROTO [A-Za-z]([A-Za-z0-9+\-.]+)+
URIHOST %{IPORHOST}(?::%{POSINT:port})?
# uripath comes loosely from RFC1738, but mostly from what Firefox
# doesn't turn into %XX
URIPATH (?:/[A-Za-z0-9$.+!*'(){},~:;=@#%&_\-]*)+
#URIPARAM \?(?:[A-Za-z0-9]+(?:=(?:[^&]*))?(?:&(?:[A-Za-z0-9]+(?:=(?:[^&]*))?)?)*)?
URIPARAM \?[A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?\-\[\]<>]*
URIPATHPARAM %{URIPATH}(?:%{URIPARAM})?
URI %{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})?

# Months: January, Feb, 3, 03, 12, December
MONTHNUM (?:0?[1-9]|1[0-2])
MONTHNUM2 (?:0[1-9]|1[0-2])
MONTHDAY (?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9])
TTY (?:/dev/(pts|tty([pq])?)(\w+)?/?(?:[0-9]+))
WINPATH (?>[A-Za-z]+:|\\)(?:\\[^\\?*]*)+
URIPROTO [A-Za-z]([A-Za-z0-9+\-.]+)+
URIHOST %{IPORHOST}(?::%{POSINT:port})?
# uripath comes loosely from RFC1738, but mostly from what Firefox
# doesn't turn into %XX
URIPATH (?:/[A-Za-z0-9$.+!*'(){},~:;=@#%&_\-]*)+
#URIPARAM \?(?:[A-Za-z0-9]+(?:=(?:[^&]*))?(?:&(?:[A-Za-z0-9]+(?:=(?:[^&]*))?)?)*)?
URIPARAM \?[A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?\-\[\]<>]*
URIPATHPARAM %{URIPATH}(?:%{URIPARAM})?
URI %{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})?

# Months: January, Feb, 3, 03, 12, December
MONTHNUM (?:0?[1-9]|1[0-2])
MONTHNUM2 (?:0[1-9]|1[0-2])
MONTHDAY (?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9])

# Years?
YEAR (?>\d\d){1,2}
HOUR (?:2[0123]|[01]?[0-9])
MINUTE (?:[0-5][0-9])
# '60' is a leap second in most time standards and thus is valid.
SECOND (?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)
TIME (?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9])
# datestamp is YYYY/MM/DD-HH:MM:SS.UUUU (or something like it)
DATE_US %{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR}
DATE_EU %{MONTHDAY}[./-]%{MONTHNUM}[./-]%{YEAR}
ISO8601_TIMEZONE (?:Z|[+-]%{HOUR}(?::?%{MINUTE}))
ISO8601_SECOND (?:%{SECOND}|60)
TIMESTAMP_ISO8601 %{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}?
DATE %{DATE_US}|%{DATE_EU}
DATESTAMP %{DATE}[- ]%{TIME}
TZ (?:[APMCE][SD]T|UTC)
DATESTAMP_RFC822 %{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ}
DATESTAMP_RFC2822 %{DAY}, %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} %{ISO8601_TIMEZONE}
DATESTAMP_OTHER %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR}
DATESTAMP_EVENTLOG %{YEAR}%{MONTHNUM2}%{MONTHDAY}%{HOUR}%{MINUTE}%{SECOND}

# Syslog Dates: Month Day HH:MM:SS
SYSLOGTIMESTAMP %{MONTH} +%{MONTHDAY} %{TIME}
PROG [\x21-\x5a\x5c\x5e-\x7e]+
SYSLOGPROG %{PROG:program}(?:\[%{POSINT:pid}\])?
SYSLOGHOST %{IPORHOST}
SYSLOGFACILITY <%{NONNEGINT:facility}.%{NONNEGINT:priority}>
HTTPDATE %{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT}

# Shortcuts
QS %{QUOTEDSTRING}

# Log formats
SYSLOGBASE %{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}:

# Log Levels
LOGLEVEL ([Aa]lert|ALERT|[Tt]race|TRACE|[Dd]ebug|DEBUG|[Nn]otice|NOTICE|[Ii]nfo|INFO|[Ww]arn?(?:ing)?|WARN?(?:ING)?|[Ee]rr?(?:or)?|ERR?(?:OR)?|[Cc]rit?(?:ical)?|CRIT?(?:ICAL)?|[Ff]atal|FATAL|[Ss]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?)

解决geoip的location不为geo_point格式

在使用geoip获取的经纬度时会遇到一个问题就是location的type是number类型的，这导致它不能被map直接使用，所以我们就需要对他的类型进行转换,需要创建一个新的模板，进行匹配。引用的原文连接 https://www.cnblogs.com/tielemao/p/13523564.html

PUT /_template/cache_log
{
  "index_patterns" : [
      "cache*" #匹配的索引
  ],
  "order" : 10,
  "mappings": {
    "properties": {
      "client_ip": {
        "type": "ip"
      },
      "geoip": {
        "dynamic": true,
        "type": "object",
        "properties": {
          "location": {
            "type": "geo_point"
          }
        }
      }
    }
  },
  "aliases" : {
      "tielemao_weblog" : { }
  }
}