ELK-logstash--到redis存取日志

搭建Redis服务

[root@test ~]# rpm -q gcc || yum -y install gcc
[root@test ~]# tar -zxvf redis-4.0.8.tar.gz   	#上传
[root@test ~]# cd redis-4.0.8/
[root@test redis-4.0.8]#  make && make install

[root@test redis-4.0.8]# ./utils/install_server.sh   //初始化,一路enter
Please select the redis port for this instance: [6379]
Selecting default: 6379
Please select the redis config file name [/etc/redis/6379.conf]
Selected default - /etc/redis/6379.conf
Please select the redis log file name [/var/log/redis_6379.log]
Selected default - /var/log/redis_6379.log
Please select the data directory for this instance [/var/lib/redis/6379]
Selected default - /var/lib/redis/6379
Please select the redis executable path [/usr/local/bin/redis-server]
Selected config:
Port           : 6379
Config file    : /etc/redis/6379.conf
Log file       : /var/log/redis_6379.log
Data dir       : /var/lib/redis/6379
Executable     : /usr/local/bin/redis-server
Cli Executable : /usr/local/bin/redis-cli
Is this ok? Then press ENTER to go on or Ctrl-C to abort.
Copied /tmp/6379.conf => /etc/init.d/redis_6379
Installing service...
Successfully added to chkconfig!
Successfully added to runlevels 345!
Starting Redis server...
Installation successful!

配置密码

[root@test ~]# vim /etc/redis/6379.conf 
bind 192.168.1.11
daemonize yes
save ""
#save 900 1
#save 300 10
#save 60 10000
requirepass root123   //配置密码

启动

[root@test ~]# /etc/init.d/redis_6379  restart
[root@test ~]# ss -ntulp | grep 6379
tcp    LISTEN     0      128    127.0.0.1:6379  *:*    users:(("redis-server",..
[root@test ~]# redis-cli
127.0.0.1:6379> get key
(error) NOAUTH Authentication required.
127.0.0.1:6379> auth 123456
OK

配置logstash

[root@logstash ~]# yum -y install logstash-2.3.4-1.noarch.rpm
[root@logstash ~]# yum -y install java
[root@logstash ~]# redis-cli -h 192.168.1.11 -p 6379
192.168.1.11:6379> auth 123456
OK

配置logstash的messegelog_to_redis.conf

[root@logstash ~]# cat messegelog_to_redis.conf
input {
  file {
    path => "/var/log/messages"
    type => "systemlog"
    start_position => "beginning"
    stat_interval => "2"
  }
}

output {
  if [type] == "systemlog" {
    redis {
      data_type => "list"
      host => "192.168.1.11"
      db => "0"
      port => "6379"
      password => "123456"
      key => "systemlog"
    }
  }
}

[root@logstash ~]# /opt/logstash/bin/logstash -f messegelog_to_redis.conf &
[1] 1549

测试存入

[root@logstash ~]# redis-cli -h 192.168.1.11 -p 6379
192.168.1.11:6379> auth 123456
OK
192.168.1.11:6379> select 0
OK
192.168.1.11:6379> llen systemlog
(integer) 9813

[root@logstash ~]# cat /etc/hosts
127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
[root@logstash ~]# cat /etc/hosts >> /var/log/messages

[root@logstash ~]# redis-cli -h 192.168.1.11 -p 6379
192.168.1.11:6379> auth 123456
OK
192.168.1.11:6379> select 0
OK
192.168.1.11:6379> llen systemlog
(integer) 9815

2.配置logstash从reids中取出数据到elasticsearch

[root@logstash2 ~]# vim redis-es.conf 
input {
  redis {
    data_type => "list"
    host => "192.168.1.11"
    db => "0"
    port => "6379"
    key => "systemlog"
    password => "123456"
  }
}

output {
  elasticsearch {
    hosts => ["192.168.1.16:9200"]
    index => "redis-systemlog-%{+YYYY.MM.dd}"
  }
}
[root@logstash2 ~]# /opt/logstash/bin/logstash -f  redis-es.conf  &