Logstash

Java :

官网下载 ：Logstash

wget https://artifacts.elastic.co/downloads/logstash/logstash-6.5.1.tar.gz

tar -zxf logstash-6.5.1.tar.gz

cd logstash-6.5.1

#bin/logstash -e 'input { stdin { } } output { stdout {} }'

vim my.conf

input{
        stdin{
                add_field => {"key"=>"value"}
                codec => "plain"
                tags => ["add"]
                type => "std"
        }
}
output{
        stdout{
                codec=>rubydebug
        }

        elasticsearch{
                hosts => "192.168.24.188:9200"
        }
}

 ./bin/logstash -f my.conf

elasticsearch :

----------------------grok

 input{
        stdin{}
}

#Grok filter plugin

#/usr/local/logstash/vendor/bundle/jruby/2.3.0/gems/logstash-patterns-core-4.1.2/patterns/grok-patterns

filter{
        grok{
                match => {
                        "message"=>"%{IP:client} %{WORD:method} %{URIPATHPARAM:request} %{NUMBER:bytes} %{NUMBER:duration}"
                }
        }
}
output{
        stdout{
#                codec=>rubydebug
        }
}

-----------------------------file-codec-json

log_format json '{'
'"client":"$remote_addr",'
'"url":"$uri",'
'"referer":"$http_referer",'
'"ua":"$http_user_agent"'
'}';

--------------[1]--[]

input{
        file{
                path => ["/home/ubuntu/test.log"]
                type => "system"
        }
        file{
                path => "/home/ubuntu/bak.log"
                type => "bak"
                codec => multiline {
                     pattern => "^\["
                     negate => true
                     what => "previous"
                }

         }
        file{
                path => "/usr/local/nginx/logs/host.access.log"
                type => "nginx"
                codec => "json"
        }
}
output{
#        stdout{
#                codec=>rubydebug
#        }
        if [type] == "system" {
                elasticsearch{
                        hosts => "192.168.1.100:9200"
                        index => "system-%{+YYYY-MM}"
                }
        }
        if [type] == "bak" {
                elasticsearch{
                        hosts => "192.168.1.100:9200"
                        index => "bak-%{+YYYY-MM}"
                }
        }
        if [type] == "nginx" {
                elasticsearch{
                        hosts => "192.168.1.100:9200"
                        index => "nginx-%{+YYYY-MM}"
                }
        }

}

----------------------------------Syslog input plugin

/etc/rsyslog.conf

*.*     @@192.168.1.100:514 # log发送到100上；

/var/log/messages # 查看系统日志；

input{
        syslog{
                type => "syslog"
                port => "514"
        }
}

output{
        stdout{
                codec=>rubydebug
        }
}
 

---------------------------Tcp input plugin

input {
        tcp {
                type => "tcp"
                host => "0.0.0.0"
                port => 12345
                mode => "server"
                ssl_enable => false
        }
}

output{
        stdout{
                codec=>rubydebug
        }
}

nc 192.168.1.100 12345 < ~/test.txt

echo 'hello world' > /dev/tcp/192.168.1.102/12345 # /dve/tcp 不存在；

--------------------------------Redis output plugin

input {
    stdin{}
}

output{
    redis {
            host => "127.0.0.1"
            port => 6379
            db => 6
            data_type => "list"
            key => "redis_log"
    }
}

./bin/logstash -f config/redis.conf  -t # 检查配置文件

[INFO ][logstash.runner          ] Using config.test_and_exit mode. Config Validation Result: OK. Exiting Logstash

--------------------Redis input plugin

input {
        redis{
                host => "127.0.0.1"
                port => 6379
                db => 6
                data_type => "list"
                key => "redis_log"
        }
}

output{
        stdout {}
}