极客时间06

1. 基于 logstash filter 功能将 nginx 默认的访问日志及 error log 转换为 json 格式并写入
2. elasticsearch 基于 logstash 收集 json 格式的 nginx 访问日志 基于 logstash 收集 java
3. 日志并实现多行合并 基于 logstash 收集 syslog 类型日志 (以 haproxy 替代网络设备) logstash
4. 收集日志并写入 Redis、再通过其它 logstash 消费至 elasticsearch 并保持 json 格式日志的解析 基于
5. docker-compose 部署单机版本 ELK

1、基于 logstash filter 功能将 nginx 默认的访问日志及 error log 转换为 json 格式并写入

input {
    file {
        path => "/var/log/nginx/access.log" 
        type => "nginx-accesslog" 
        stat_interval => "1" 
        start_position => "beginning" 
    }
    file {
        path => "/var/log/nginx/error.log"
        type => "nginx-errorlog"
        stat_interval => "1" 
        start_position => "beginning"
    }
}

filter {
    if [type] == "nginx-accesslog" {
        grok {
            match => { "message" => ["%{IPORHOST:clientip} - %{DATA:username} \[%{HTTPDATE:request-time}\] \"%{WORD:request-method} %{DATA:request-uri} HTTP/%{NUMBER:http_version}\" %{NUMBER:response_code} %{NUMBER:body_sent_bytes} \"%{DATA:referrer}\" \"%{DATA:useragent}\""] }
            remove_field => "message"
            add_field => {"project" => "nginx" }
        }
        mutate {
            convert => [ "[response_code]", "integer"]
        }
    }
    if [type] == "nginx-errorlog" {
        grok {
            match => { "message" => ["(?<timestamp>%{YEAR}[./]%{MONTHNUM}[./]%{MONTHDAY} %{TIME}) \[%{LOGLEVEL:loglevel}\] %{POSINT:pid}#%{NUMBER:threadid}\: \*%{NUMBER:connectionid} %{GREEDYDATA:message}, client: %{IPV4:clientip}, server: %{GREEDYDATA:server}, request: \"(?:%{WORD:request-method} %{NOTSPACE:request-uri}(?: HTTP/%{NUMBER:httpversion}))\", host: %{GREEDYDATA:domainname}"]}
            remove_field => "message"
        }
    }
}
output {
    if [type] == "nginx-accesslog" {
        elasticsearch {
            hosts => ["192.168.80.85:9200"]
            index => "nginx-accesslog-%{+yyyy.MM.dd}"
            user => "serveradmin"
            password => "server123"
        }
    }
    if [type] == "nginx-error.log" {
        elasticsearch {
            hosts => ["192.168.80.85:9200"]
            index => "nginx-errorlog-%{+yyyy.MM.dd}"
            user => "serveradmin"
            password => "server123"
        }
    }
}

2. elasticsearch 基于 logstash 收集 json 格式的 nginx 访问日志 基于 logstash 收集 java

input {
  file {
    path => "/var/log/nginx/access.log"
    start_position => "end"
    type => "nginx-json-accesslog"
    stat_interval => "1"
    codec => json
  }
}


output {
  if [type] == "nginx-json-accesslog" {
    elasticsearch {
      hosts => ["192.168.80.85:9200"]
      index => "nginx-accesslog-%{+YYYY.MM.dd}"
      user => "serveradmin"
      password => "server123"
  }}
}