Filebeat+Logstash+Elasticsearch抓取日志

环境：

Nginx：10.10.36.126:8000

Filebeat：10.10.36.126

Logstash：10.10.36.128:5044

Elasticsearch：10.10.36.128:9200

1、安装Nginx

sudo yum install pcre pcre-devel zlib zlib-devel openssl openssl-devel

tar -zxvf nginx-1.11.8.tar.gz

cd nginx-1.11.8

./configure --prefix=/home/wangzi/Downloads/nginx

make && make install

cd ../nginx

vi conf/nginx.conf

user  root;

# 自定义日志格式

log_format json '{"@timestamp":"$time_iso8601",'
                 '"slbip":"$remote_addr",'
                 '"clientip":"$http_x_forwarded_for",'
                 '"serverip":"$server_addr",'
                 '"size":$body_bytes_sent,'
                 '"responsetime":$request_time,'
                 '"domain":"$host",'
                 '"method":"$request_method",'
                 '"requesturi":"$request_uri",'
                 '"url":"$uri",'
                 '"appversion":"$HTTP_APP_VERSION",'
                 '"referer":"$http_referer",'
                 '"agent":"$http_user_agent",'
                 '"status":"$status",'
                 '"devicecode":"$HTTP_HA"}';

listen 8000;

access_log  /var/log/host.access.log  json;

启动：sudo ./sbin/nginx 

访问http://10.10.36.126:8000/可以看到Nginx主页面

2、安装Elasticsearch

sudo rpm -ivh elasticsearch-5.2.2.rpm

# ES 会被默认安装在 /usr/share/elasticsearch/

# ES配置文件在/etc/elasticsearch/

# ES日志文件在/var/log/elasticsearch/

# 配置系统最大打开文件描述符数
sudo vi /etc/sysctl.conf
fs.file-max=65535
# 配置进程最大打开文件描述符
sudo vi /etc/security/limits.conf
# End of file
* soft nofile 65535
* hard nofile 65535
# 配置 JVM内存
sudo vi /etc/sysconfig/elasticsearch
ES_HEAP_SIZE=4g

启动： sudo systemctl start elasticsearch

执行 curl 'http://10.10.36.128:9200/_search?pretty' 可以看到ES中的所有数据

# 删除数据：curl -XDELETE 'http://10.10.36.128:9200/indexName'

3、安装Logstash

方式一：rpm包安装

sudo rpm -ivh logstash-5.2.2.rpm

# 默认安装在 /usr/share/logstash/

# 配置文件在/etc/logstash/

# 日志文件在/var/log/logstash/

配置监听5044端口：

sudo vi /etc/logstash/conf.d/filebeat_logstash_es.conf

input {
  beats {
    port => 5044
  }
}
output {
  elasticsearch {
    hosts => "10.10.36.128:9200"
    manage_template => false
    index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
    document_type => "%{[@metadata][type]}"
  }
}

启动：sudo systemctl start logstash

方式二：tar包安装

tar -zxvf logstash-5.2.2.tar.gz

cd logstash-5.2.2

vi config/filebeat_logstash_es.conf

input {
  beats {
    port => 5044
  }
}
output {
  elasticsearch {
    hosts => "10.10.36.128:9200"
    manage_template => false
    index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
    document_type => "%{[@metadata][type]}"
  }
}

验证配置文件：./bin/logstash -f ./config/filebeat_logstash_es.conf -t

启动：./bin/logstash -f ./config/filebeat_logstash_es.conf

4、安装Filebeat

sudo rpm -ivh filebeat-5.2.2-x86_64.rpm

# 默认安装在 /usr/share/filebeat/
# 配置文件在/etc/filebeat/
# 日志文件在/var/log/filebeat/

修改配置文件：

vi /etc/filebeat/filebeat.yml

paths:
    - /var/log/host.access.log

output.logstash:
  # The Logstash hosts
  hosts: ["10.10.36.128:5044"]

修改ES的索引模版

curl -XPUT 'http://10.10.36.128:9200/_template/filebeat' -d@/etc/filebeat/filebeat.template.json

启动：sudo systemctl start filebeat

此时，执行curl 'http://10.10.36.128:9200/_search?pretty' 可以看到Nginx输出的日志