syslog、filebeat+Logstash +redis+Logstash+elasticsearch

要采集分析监控设备日志,日志来源由syslog和本地.log日志提供

具体安装看上一篇

一、logstash配置

logstash-7.3.0目录下新建文件filebeat-pipeline.conf和redis-elasticsearch-pipeline.conf

filebeat-pipeline.conf 内容

input {
     #beats{
       type => "system-filebeat"
       port => 5044
  }

     syslog{
        type => "system-syslog"
        host => "syslog汇聚地址"
        port => "514"
    }
}
#filter {
#}
output{
       if[type] == "system-filebeat"{
                redis {
                data_type => "list"
                host => "redisip地址"
                db => "1"
                port => "6379"
#               password => "123456"
                key => "syslogsystemlog"
                }
       # }
        if[type] == "system-syslog"{
                redis {
                data_type => "list"
                host => "redisip地址"
                db => "1"
                port => "6379"
#               password => "123456"
                key => "syslogsystemlog"
                }

        }
}

redis-elasticsearch-pipeline.conf内容

input {
    redis {
        data_type => "list"
        host => "127.0.0.1"
        db => "1"
        port => "6379"
        key => "syslogsystemlog"
#        password => "123456"
    }
}

output {
    elasticsearch {
        hosts => ["127.0.0.1:9200"]
        index => "test-syslog-ridis-systemlog-%{+YYYY.MM.dd}"
    }
}

logstash多实例启动

//启动多个实例(实例目录conf/)
bin/logstash -f conf/ --config.reload.automatic --path.data=/opt/collectlog/logstash

//启动多个实例(实例目录conf/)后台启动
nohup bin/logstash -f conf/ --config.reload.automatic --path.data=/opt/collectlog/logstash &

二、验证

启动所有程序

1、redis验证

##启动redis客户端
(venv) [root@localhost bin]# ./redis-cli
##进入编号为1的redis库(数据保存在1库,上面配置)
redis 127.0.0.1:6379> SELECT 1
OK
##查看redis是否保存key值
redis 127.0.0.1:6379[1]> KEYS *
1) "syslogsystemlog"
##查看key中的vlaue (elasticsearch消费完后redis删除)
redis 127.0.0.1:6379[1]> LPOP syslogsystemlog
"{\"host\":\"221.130.213.64\",\"timestamp\":\"Aug  9 10:02:01\",\"message\":\"187 Base SYSTEM-WARNING-tmnxStateChange-2009 [LDP]:  Status of vRtrLdpSessionTable: Virtual Router 1, Peer 221.130.208.1:0. changed administrative state: inService, operational state: inService\\n\",\"type\":\"system-syslog\",\"priority\":188,\"severity\":4,\"facility_label\":\"local7\",\"logsource\":\"221.130.213.64\",\"@version\":\"1\",\"@timestamp\":\"2019-08-09T02:02:01.000Z\",\"severity_label\":\"Warning\",\"facility\":23,\"program\":\"TMNX\"}"
redis 127.0.0.1:6379[1]> 

2、elasticsearch验证

##查看所有的索引
(venv) [root@localhost bin]# curl 'localhost:9200/_cat/indices?v'
health status index                                  uuid                   pri rep docs.count docs.deleted store.size pri.store.size
yellow open   test-syslog-ridis-systemlog-2019.08.09 sou5xsiUR6GfTfZFV-4Q9g   1   1     921782            0    172.7mb        172.7mb
yellow open   syslog-edis-systemlog-2019.08.09       b6p2yGnYQU-0jEC1e1rWHQ   1   1       2362            0    318.5kb        318.5kb
yellow open   test-syslog-ridis-systemlog-2019.08.08 QSTQmF3aTBWSr8oiA-fZ0A   1   1    2039369            0    231.3mb        231.3mb
##查看test-syslog-ridis-systemlog-2019.08.09索引中的所有数据
(venv) [root@localhost bin]#  curl 'localhost:9200/test-syslog-ridis-systemlog-2019.08.09/_search?q=*&pretty'

##删除索引
curl -XDELETE 'localhost:9200/test-syslog-ridis-systemlog-2019.08.09?pretty'

以上执行完,都有东西句OK了