logstash 收集日志并写入 redis

1、 安装reids

1.1 安装配置

apt install redis -y
vim /etc/redis/redis.conf
bind 0.0.0.0
requirepass 123456

1.2 启动验证

systemctl restart redis
root@ubuntu-node1:~# redis-cli 
127.0.0.1:6379> AUTH 123456
OK
127.0.0.1:6379> ping
PONG

2、配置logstash 将日志写入至 redis

2.1 添加logstash配置文件

vim /etc/logstash/conf.d/tomcat_to_redis.conf
input {
  file {
    path => "/apps/tomcat/logs/localhost_access_log.*.log"
    type => "tomcat-redis-log"
    start_position => "beginning"
    stat_interval => "2"
    codec => "json"
  }
}
output {
  if [type] == "tomcat-redis-log" {
    redis {
      #key类型
      data_type =>"list"
      #key名称
      key => "tomcat-redis-log"
      host => "10.10.100.116"
      port => "6379"
      db => "0"
      password =>"123456"
    }
  }
}

2.1 启动查看

#启动
systemctl restart redis

#登入redis查看
127.0.0.1:6379> KEYS *
1) "tomcat-redis-log"

3、配置其他 logstash 服务器从 redis 读取数据

"/etc/logstash/conf.d/redis-to-es.conf" 21L, 393C                                                                                                                                              18,28         All
input {
  redis {
      #key类型
      data_type =>"list"
      #key名称
      key => "tomcat-redis-log"
      host => "10.10.100.116"
      port => "6379"
      db => "0"
      password =>"123456"
      codec => "json"
  }
}
output {
  if [type] == "tomcat-redis-log" {
    elasticsearch {
      hosts => ["10.10.100.120:9200"]
      index => "tomcat-redis-100.115-%{+YYYY.MM.dd}"
    }
}
}

#启动logstash 
systemctl start logstash

3.1 验证 redis 的数据是否被取出

3.2 查看索引

3.3 kibana 查看日志