ELK 通过 Logstash 收集日志并写入 Redis

用一台服务器按照部署 redis 服务，专门用于日志缓存使用，用于 web 服务器 产生大量日志的场景，例如下面的服务器内存即将被使用完毕，查看是因为 redis 服务保存了大量的数据没有被读取而占用了大量的内存空间。

一、部署并且启动 redis 服务

root@redis:~# apt -y install redis
root@redis:~# vim /etc/redis/redis.conf
bind 0.0.0.0
save ""
#save 900 1
#save 300 10
#save 60 10000
requirepass 123456
root@redis:~# systemctl restart redis
root@redis:~# systemctl enable redis
root@redis:~# ss -ntl
State                    Recv-Q                     Send-Q                                          Local Address:Port                                         Peer Address:Port
LISTEN                   0                          128                                             127.0.0.53%lo:53                                                0.0.0.0:*
LISTEN                   0                          128                                                   0.0.0.0:22                                                0.0.0.0:*
LISTEN                   0                          128                                                 127.0.0.1:6011                                              0.0.0.0:*
LISTEN                   0                          128                                                   0.0.0.0:6379                                              0.0.0.0:*
LISTEN                   0                          128                                                      [::]:22                                                   [::]:*
LISTEN                   0                          128                                                     [::1]:6011                                                 [::]:*
root@redis:~# redis-cli
127.0.0.1:6379> AUTH 123456
OK

二、配置WEB服务器的日志写入redis

root@web1:/etc/logstash/conf.d# vim nginx-log-to-redis.conf
input {
  file {
    path => "/var/log/nginx/access.log"
    type => "nginx-accesslog"
    start_position => "beginning"
    stat_interval => "3 second"
    codec => "json"
  }
  file {
    path => "/apps/nginx/logs/error.log"
    type => "nginx-errorlog"
    start_position => "beginning"
    stat_interval => "3 second"
  }
}

output {
  if [type] == "nginx-accesslog" {
    redis {
      data_type => "list"
      key => "lck-nginx-accesslog"
      host => "10.0.0.34"
      port => "6379"
      db => "1"
      password => "123456"
    }
  }
  if [type] == "nginx-errorlog" {
    redis {
      data_type => "list"
      key => "lck-nginx-errorlog"
      host => "10.0.0.34"
      port => "6379"
      db => "1"
      password => "123456"
    }
  }
}

三、检测Logstash配置文件语法是否正确

root@web1:~# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/nginx-log-to-redis.conf -t

四、启动服务并验证

root@web1:~# systemctl restart logstash.service

尝试访问页面，生成日志

root@redis:~# redis-cli
127.0.0.1:6379> AUTH 123456
OK
127.0.0.1:6379> SELECT 1
OK
127.0.0.1:6379[1]> KEYS *
1) "lck-nginx-errorlog"
2) "lck-nginx-accesslog"
127.0.0.1:6379[1]> RPOP lck-nginx-errorlog
"{\"message\":\"2021/08/29 05:07:32 [error] 79879#0: *10 open() \\\"/apps/nginx/html/favicon.ico\\\" failed (2: No such file or directory), client: 10.0.0.1, server: localhost, request: \\\"GET /favicon.ico HTTP/1.1\\\", host: \\\"10.0.0.37\\\", referrer: \\\"http://10.0.0.37/\\\"\",\"path\":\"/apps/nginx/logs/error.log\",\"type\":\"nginx-errorlog\",\"@version\":\"1\",\"@timestamp\":\"2021-08-29T05:07:33.254Z\",\"host\":\"0.0.0.0\"}"
127.0.0.1:6379[1]> RPOP lck-nginx-accesslog
"{\"path\":\"/var/log/nginx/access.log\",\"type\":\"nginx-accesslog\",\"@timestamp\":\"2021-08-29T05:07:32.000Z\",\"http_host\":\"10.0.0.37\",\"host\":\"10.0.0.37\",\"size\":555,\"url\":\"/favicon.ico\",\"status\":\"404\",\"referer\":\"http://10.0.0.37/\",\"responsetime\":0.0,\"upstreamtime\":\"-\",\"xff\":\"-\",\"domain\":\"10.0.0.37\",\"@version\":\"1\",\"clientip\":\"10.0.0.1\",\"upstreamhost\":\"-\"}"

五、配置其他 logstash 服务器从 redis 读取数据并传到 elasticsearch

10.0.0.35 安装 logstash

apt install -y openjdk-8-jdk

# 将 logstash-7.12.1-amd64.deb 软件包传到 /usr/local/src 目录下，并进行安装
dpkg -i /usr/local/src/logstash-7.12.1-amd64.deb

修改配置文件

root@ubuntu1804:~# vim /etc/logstash/conf.d/redis-to-es.conf
input {
  redis {
    data_type => "list"
    key => "lck-nginx-accesslog"
    host => "10.0.0.34"
    port => "6379"
    db => "1"
    password => "123456"
    threads => "4"
  }
  redis {
    data_type => "list"
    key => "lck-nginx-errorlog"
    host => "10.0.0.34"
    port => "6379"
    db => "1"
    password => "123456"
    threads => "4"
  }
}

output {
  if [type] == "nginx-accesslog" {
    elasticsearch {
      hosts => ["10.0.0.31:9200"]
      index => "nginx-newindex-accesslog-%{+YYYY.MM.dd}"
    }
  }
  if [type] == "nginx-errorlog" {
    elasticsearch {
      hosts => ["10.0.0.31:9200"]
      index => "nginx-newindex-errorlog-%{+YYYY.MM.dd}"
    }
  }
}

六、检测Logstash配置文件语法是否正确

/usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/redis-to-es.conf -t

七、启动服务并验证

systemctl restart logstash.service

我们发现redis的数据已经全部被logstash取走，数据已经变成空的了

127.0.0.1:6379[1]> KEYS *
(empty list or set)

八、创建索引方便查询日志

8.1 创建访问日志索引

8.2 创建错误日志索引