5、ELK+Redis

一、Redis安装

1、下载安装包解压安装

yum install -y wget net-tools gcc gcc-c++ make tar openssl openssl-devel cmake
cd /usr/local/
wget 'http://download.redis.io/releases/redis-4.0.9.tar.gz'
tar -zxf redis-4.0.9.tar.gz
cd redis-4.0.9/
make
mkdir -pv /usr/local/redis/conf /usr/local/redis/bin
cp src/redis* /usr/local/redis/bin/
cp redis.conf /usr/local/redis/conf

2、修改配置文件
vim /usr/local/redis/conf/redis.conf

daemonize yes
dir /tmp
bind 0.0.0.0
requirepass wangxiaoyu

3、启动redis

/usr/local/redis/bin/redis-server /usr/local/redis/conf/redis.conf

4、redis简单操作

[root@elk-node2-51 redis-4.0.9]# /usr/local/redis/bin/redis-cli  
127.0.0.1:6379> auth wangxiaoyu
OK
127.0.0.1:6379> set name wangxiaoyu
OK
127.0.0.1:6379> get name
"wangxiaoyu"

二、Filebeat日志写入redis

1、修改filebeat配置文件然后重启filebeat，将output改为redis的地址

[root@elk-node2-51 redis-4.0.9]# cat /usr/local/filebeat-6.6.0/filebeat.yml
filebeat.inputs:
- type: log
  tail_files: true
  backoff: "1s"
  paths:
      - /usr/local/nginx/logs/access.log
  fields:
    type: access
  fields_under_root: true

- type: log
  tail_files: true
  backoff: "1s"
  paths:
      - /var/log/secure
  fields:
    type: secure
  fields_under_root: true

output:
  redis:
      hosts: ["10.0.0.51"]
      port: 6379
      password: 'wangxiaoyu'
      key: 'access'

2、查看redis存储的日志信息
访问nginx页面，然后登陆redis查看
有一个list类型名称为access的key，可以看到有保存的日志信息

[root@elk-node2-51 redis-4.0.9]# /usr/local/redis/bin/redis-cli 
127.0.0.1:6379> auth wangxiaoyu
127.0.0.1:6379> keys *
1) "name"
2) "access"
127.0.0.1:6379> type access
list

127.0.0.1:6379> lrange access 0 -1
1) "{\"@timestamp\":\"2019-05-08T03:07:34.725Z\",\"@metadata\":{\"beat\":\"filebeat\",\"type\":\"doc\",\"version\":\"6.6.0\"},\"offset\":28297,\"log\":{\"file\":{\"path\":\"/usr/local/nginx/logs/access.log\"}},\"message\":\"10.0.0.1 - - [08/May/2019:11:07:27 +0800] \\\"GET / HTTP/1.1\\\" 304 0 \\\"-\\\" \\\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36\\\"\",\"source\":\"/usr/local/nginx/logs/access.log\",\"prospector\":{\"type\":\"log\"},\"host\":{\"name\":\"elk-node2-51\"},\"input\":{\"type\":\"log\"},\"type\":\"access\",\"beat\":{\"hostname\":\"elk-node2-51\",\"version\":\"6.6.0\",\"name\":\"elk-node2-51\"}}"
2) "{\"@timestamp\":\"2019-05-08T03:07:34.727Z\",\"@metadata\":{\"beat\":\"filebeat\",\"type\":\"doc\",\"version\":\"6.6.0\"},\"source\":\"/usr/local/nginx/logs/access.log\",\"log\":{\"file\":{\"path\":\"/usr/local/nginx/logs/access.log\"}},\"message\":\"10.0.0.1 - - [08/May/2019:11:07:28 +0800] \\\"GET / HTTP/1.1\\\" 304 0 \\\"-\\\" \\\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36\\\"\",\"prospector\":{\"type\":\"log\"},\"beat\":{\"hostname\":\"elk-node2-51\",\"version\":\"6.6.0\",\"name\":\"elk-node2-51\"},\"offset\":28484,\"input\":{\"type\":\"log\"},\"type\":\"access\",\"host\":{\"name\":\"elk-node2-51\"}}"

3、修改Logstash配置文件从Redis中读取数据
将input改为redis的相关信息，然后重启logstash

[root@elk-node2-51 redis-4.0.9]# cat /usr/local/logstash-6.6.0/config/logstash.conf
input {
  redis {
    host => '10.0.0.51'
    port => 6379
    key => "access"
    data_type => "list"
    password => 'wangxiaoyu'
  }
}

filter {
  mutate {
    rename => { "[host][name]" => "host" }
  }
}

output{

  if [type] == "access" {
    elasticsearch {
      hosts => ["http://10.0.0.50:9200"]
      index => "access-%{+YYYY.MM.dd}"
    }
  }

  if [type] == "secure" {
    elasticsearch {
      hosts => ["http://10.0.0.50:9200"]
      index => "secure-%{+YYYY.MM.dd}"
    }
  }

}

4、然后查看有日志