logstash过滤apache的日志到数据库（redis）再到elasticsearch

server1：elasticsearch
server2：httpd，logstash
server3：redis

1.在server3上安装redis并且开启redis

[root@server3 ~]# ls
redis-5.0.3.tar.gz
[root@server3 ~]# tar zxf redis-5.0.3.tar.gz 
[root@server3 ~]# cd redis-5.0.3/
[root@server3 redis-5.0.3]# yum install -y gcc
[root@server3 redis-5.0.3]# yum install -y make
[root@server3 redis-5.0.3]# make
[root@server3 redis-5.0.3]# make install
[root@server3 redis-5.0.3]# cd utils/
[root@server3 utils]# ./install_server.sh 
[root@server3 redis-5.0.3]# vim /etc/redis/6379.conf
70 bind 0.0.0.0
[root@server3 redis-5.0.3]# /etc/init.d/redis_6379 restart

2.在server2上编辑logstash的文件

server2上是过滤httpd日志指向redis

[root@server2 conf.d]# vim es.conf 
input {
#         stdin {}
#          file {
#                  path => "/var/log/elasticsearch/my-es.log"
#                  start_position => "beginning"
#                  codec => multiline {
#                           pattern => "^\["
#                           negate => "true"
#                           what => "previous"
#                }
#          }
#
#           syslog {
#                     port => 514
#            }
             file {
                      path => "/var/log/httpd/access_log"
                      start_position => "beginning"
             }
}

filter {
  grok {
    match => { "message" => "%{HTTPD_COMBINEDLOG}" }
  }
}

output {
       stdout {}
        redis {
                host => ["172.25.42.3:6379"]
                data_type => "list"
                key => logstashtoredis
        }
}

3.在server2上执行

[root@server2 conf.d]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/es.conf 

4.在server3上安装logstash

[root@server2 ~]# ls
logstash-6.6.1.rpm
[root@server2 ~]# yum install -y ruby
[root@server2 ~]# rpm -ivh logstash-6.6.1.rpm
[root@server2 ~]# /usr/share/logstash/bin/logstash -e 'input { stdin { } } output { stdout {} }'     设置输入输出为标准终端目录

5.在server3上编辑logstash的相应文件通过将redis的信息过滤到elasticsearch

[root@server3 conf.d]# pwd
/etc/logstash/conf.d
[root@server3 conf.d]# vim redis.conf 
input {
        redis {
              host => ["172.25.42.3"]
              data_type => "list"
              key => "logstashtoredis"
        }
}


output {
        stdout {}

        elasticsearch {
             hosts => ["172.25.42.1:9200"]
             index => "apachelog-%{+YYYY.MM.dd}"
         }
}

6.在server3上执行查看

[root@server3 conf.d]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/redis.conf
刷新网页查看即可