elk 简单搭建方法

最新推荐文章于 2024-04-11 18:37:33 发布

my_bai

最新推荐文章于 2024-04-11 18:37:33 发布

阅读量548

点赞数

分类专栏： ELK 之nginx 日志分析

本文链接：https://blog.csdn.net/my_bai/article/details/66971546

版权

ELK 之nginx 日志分析专栏收录该内容

5 篇文章 0 订阅

订阅专栏

下载 elastic.co key文件

rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch

添加yum源，如elastic.repi

[logstash-5.x]

name=Elastic repository for 5.x packages
baseurl=https://artifacts.elastic.co/packages/5.x/yum
gpgcheck=1
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=1
autorefresh=1
type=rpm-md

下载

yum -y install elasticsearch

yum -y install logstash

yum -y install kibana

/etc/init.d/elasticsearch start

/etc/init.d/elasticsearch 修改

LOG_DIR="/data0/elasticsearch/log"
#DATA_DIR="/var/lib/elasticsearch"
DATA_DIR="/data0/elasticsearch/lib"

补：启动异常：ERROR: bootstrap checks failed
system call filters failed to install; check the logs and fix your configuration or disable system call filters at your own risk

问题原因：因为Centos6不支持SecComp，而ES5.2.1默认bootstrap.system_call_filter为true进行检测，所以导致检测失败，失败后直接导致ES不能启动。详见：https://github.com/elastic/elasticsearch/issues/22899

解决方法：在elasticsearch.yml中配置bootstrap.system_call_filter为false，注意要在Memory下面:
bootstrap.memory_lock: false
bootstrap.system_call_filter: false

/etc/init.d/kibana/ start

nohup /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/nginx_logstash.conf &>/dev/null &

nginx_logstash.conf

#标准输入
input {
file {
#日志路径
path => ["/data0/logs/sns_live.log"]
#日志类型
type => "nginx_access"
start_position => "beginning"
}
}
#正则匹配
filter {
grok {
match => {"message"=> "%{IPORHOST:client_ip} (%{USER:ident}|-) (%{USER:auth}|-) \[%{HTTPDATE:timestamp}\] \"%{WORD:http_verb} %{URIPATH:baseurl}(?:\?%{NOTSPACE:request}|) HTTP/%{NUMBER:http_version}\" %{NUMBER:http_status_code} (?:%{NUMBER:bytes_read}|-) \"%{NUMBER:time_duration:float}\" (%{NOTSPACE:request_body}|-) (%{QS:request_referer}|-) %{QS:agent} (%{NOTSPACE:x_forward}|-) %{WORD:scheme} %{NUMBER:request_lenght} (%{IPORHOST:http_host}|-)"}
}
kv {
field_split => "&" #切割符
include_keys => ["user_id","imei"] #要的字段
allow_duplicate_values => "true" #允许values重复
source => "request" #来源字段
#default_keys => [ "user_id","0000000"] #没有匹配的默认值
}
}
urldecode {
all_fields => true
}
date {
locale => "en"
match => ["timestamp" , "dd/MMM/YYYY:HH:mm:ss Z"]
}

}
#标准输出
output {
stdout { codec => rubydebug }
elasticsearch {
hosts => ["127.0.0.1:9200"]
codec => "json"
index => "logstash-nginx-%{+YYYY.MM.dd}"
}
}