《Centos7——logstash+redis消息队列+elasticsrach+kibana》

logstash+redis消息队列+elasticsrach+kibana部署

环境准备
3台Centos7
192.168.234.130 jdk+logstah
192.168.234.131 redis+logstash+jdk
192.168.234.132 jdk+elasticsearch+kibana

第一台操作(192.168.234.130)

1. 安装Nginx

https://blog.csdn.net/weixin_45842014/article/details/109740721

[root@localhost ~]# /usr/local/nginx/sbin/nginx

2. 安装logstash

[root@localhost ~]#  systemctl stop firewalld;setenforce 0

[root@localhost ~]#  rpm -ivh jdk-8u131-linux-x64_.rpm

[root@localhost ~]#  rpm -ivh logstash-7.3.2.rpm

3. 将日志输出给redis

[root@localhost ~]# cd /etc/logstash/conf.d/
[root@localhost ~]# vim log.conf
input {
        file {
                path => "/var/log/messages"
                type => "system-log"
                start_position => "beginning"
        }
        file {
                path => "/usr/local/nginx/logs/*.log"
                type => "nginx-log"
                start_position => "beginning"
        }
}
output {
        if [type] == "system-log" {
                redis {
                        host => "192.168.234.131:6379"
                        data_type => "list"
                        key => "logstash:system-log"
                }
        }
        if [type] == "nginx-log" {
                redis {
                        host => "192.168.234.131:6379"
                        data_type => "list"
                        key => "logstash:nginx-log"
                }
        }
}

4. 启动logstash

[root@localhost ~]# chmod +r /var/log/messages

[root@localhost ~]# systemctl start logstash

[root@localhost ~]# systemctl enable logstash

第二台操作

1. 安装logstash

[root@localhost ~]# systemctl stop firewalld;setenforce 0

[root@localhost ~]# rpm -ivh jdk-8u131-linux-x64_.rpm

[root@localhost ~]# rpm -ivh logstash-7.3.2.rpm

2. 从redis读取日志数据，并输出给elasticsearch

[root@localhost ~]# vim /etc/logstash/conf.d/log.conf
input {
        redis {
                host => "192.168.234.131"
                port => "6379"
                type => "system-log"
                data_type => "list"
                key => "logstash:system-log"
        }
        redis {
                host => "192.168.234.131"
                port => "6379"
                type => "nginx-log"
                data_type => "list"
                key => "logstash:nginx-log"
        }
}
output {
        if [type] == "system-log" {
                elasticsearch {
                        hosts => "192.168.234.132:9200"
                        index => "system-log-1804a"
                }
        }
        if [type] == "nginx-log" {
                elasticsearch {
                        hosts => "192.168.234.132:9200"
                        index => "nginx-log-1804a"
                }
        }
}

3. 启动logstash

[root@localhost ~]# systemctl start logstash

[root@localhost ~]# systemctl enable logstash

4. 安装redis

[root@localhost ~]# yum -y install gcc gcc-c++

[root@localhost ~]# tar xzf redis-3.2.11.tar.gz

[root@localhost ~]# cd redis-3.2.11

[root@localhost ~]# make && make install

[root@localhost ~]# cp redis.conf /etc

5. 修改redis配置文件

[root@localhost ~]# vim  /etc/redis.conf

[root@localhost redis-3.2.11]# cat /etc/redis.conf |egrep '^bind|^daemon'
bind 0.0.0.0
daemonize yes

6. 启动redis

[root@localhost ~]# redis-server /etc/redis.conf

第三台操作

1. 安装elasticsearch和kibana

[root@localhost ~]# systemctl stop firewalld;setenforce 0
[root@localhost ~]# rpm -ivh jdk-8u131-linux-x64_.rpm
[root@localhost ~]# rpm -ivh elasticsearch-7.3.2-x86_64.rpm kibana-7.3.2-x86_64.rpm

2. 修改elasticsearch配置文件

[root@localhost ~]# vim /etc/elasticsearch/elasticsearch.yml
[root@localhost ~]# cat /etc/elasticsearch/elasticsearch.yml|egrep -v '^$|^#'
cluster.name: my-application
node.name: node-1
path.data: /var/lib/elasticsearch
path.logs: /var/log/elasticsearch
network.host: 192.168.234.132
http.port: 9200
discovery.seed_hosts: ["192.168.234.132"]
cluster.initial_master_nodes: ["node-1"]

3. 启动elasticsearch

[root@localhost ~]# systemctl start elasticsearch

[root@localhost ~]# systemctl enable elasticsearch

4. 修改kibana配置文件

[root@localhost ~]# vim /etc/kibana/kibana.yml
[root@localhost ~]# cat /etc/kibana/kibana.yml |egrep -v '^$|^#'
server.port: 5601
server.host: "192.168.234.132"
elasticsearch.hosts: ["http://192.168.234.132:9200"]

5. 启动kibana

[root@localhost ~]# systemctl start kibana

[root@localhost ~]# systemctl enable kibana

6. 查看索引

[root@localhost ~]# [root@localhost ~]# curl -X GET http://192.168.234.132:9200/_cat/indices?v
health status index                uuid                   pri rep docs.count docs.deleted store.size pri.store.size
green  open   .kibana_task_manager olFgJnOgSQG7xqr6PlMRcg   1   0          2            0     45.5kb         45.5kb
yellow open   nginx-log-1804a      zKnxnoT0SHaZ6XY4zo57mA   1   1          7            0     10.1kb         10.1kb
yellow open   system-log-1804a     O0cDAU54QOSsAi1_xmy7eg   1   1       2196            0    483.6kb        483.6kb
green  open   .kibana_1            D8Jlq9lEQAWV8Kk7udZ_Nw   1   0          2            0     11.3kb         11.3kb

7. 登录网页(192.168.234.132:9200)