ELK 安装

Elasticsearch安装

[root@izbp1enj8meljjp5l7yj7ez ~]# java -version

openjdk version "1.8.0_181"

OpenJDK Runtime Environment (build 1.8.0_181-b13)

OpenJDK 64-Bit Server VM (build 25.181-b13, mixed mode)

###签名秘钥，使系统免收软件包的欺骗

[root@izbp1enj8meljjp5l7yj7ez ~]# rpm --import https://packages.elastic.co/GPG-KEY-elasticsearch

###创建 yum 源文件

[root@izbp1enj8meljjp5l7yj7ez ~]# vim /etc/yum.repos.d/elasticsearch.repo

[elasticsearch-7.x]

name=Elasticsearch repository for 7.x packages

baseurl=https://artifacts.elastic.co/packages/7.x/yum

gpgcheck=1

gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch

enabled=1

autorefresh=1

type=rpm-md

[root@izbp1enj8meljjp5l7yj7ez ~]# yum search elastic

elasticsearch.x86_64 : Elasticsearch is a distributed RESTful search engine built for the cloud. Reference documentation

: can be found at https://www.elastic.co/guide/en/elasticsearch/reference/current/index.html and the

: 'Elasticsearch: The Definitive Guide' book can be found at

: https://www.elastic.co/guide/en/elasticsearch/guide/current/index.html

heartbeat-elastic.i686 : Ping remote services for availability and log results to Elasticsearch or send to Logstash.

heartbeat-elastic.x86_64 : Ping remote services for availability and log results to Elasticsearch or send to Logstash.

[root@izbp1enj8meljjp5l7yj7ez ~]# yum install -y elasticsearch.x86_64

[root@izbp1enj8meljjp5l7yj7ez ~]# vim /etc/elasticsearch/elasticsearch.yml

network.host:localhost

http.port: 9200

Kibana 安装

[root@izbp1enj8meljjp5l7yj7ez ~]# yum install -y kibana

[root@izbp1enj8meljjp5l7yj7ez ~]# vim /etc/kibana/kibana.yml

server.port: 5601

# Specifies the address to which the Kibana server will bind. IP addresses and host names are both valid values.

# The default is 'localhost', which usually means remote machines will not be able to connect.

# To allow connections from remote users, set this parameter to a non-loopback address.

server.host: "0.0.0.0"

 

一般来说 filebeats可以将数据直接发送到 elasticsearch ,先建议 logstash 进行 收集数据处理，将不同来源的数据,转换成通用的格式，再将其导入到elasticsearch ;

[root@izbp1enj8meljjp5l7yj7ez ~]# yum install -y logstash

[root@izbp1enj8meljjp5l7yj7ez ~]# vim /etc/logstash/conf.d/02-beats-input.conf ###监听5044端口的输入

input {

beats {

port => 5044

}

}

[root@izbp1enj8meljjp5l7yj7ez ~]# vim 10-syslog-filter.conf

filter {

grok {

match => { "message" => "%{IP:client_id_address} %{WORD:method} %{URIPATHPARAM:request} %{NUMBER:bytes} %{NUMBER:http_response_time}" }

}

}

[root@izbp1enj8meljjp5l7yj7ez ~]# vim /etc/logstash/conf.d/30-elasticsearch-output.conf ###监听5044端口的输入

output {

elasticsearch {

hosts => ["localhost:9200"]

manage_template => false

index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"

}

}

[root@izbp1enj8meljjp5l7yj7ez ~]# systemctl start logstash

[root@izbp1enj8meljjp5l7yj7ez ~]# systemctl enable logstash

Created symlink from /etc/systemd/system/multi-user.target.wants/logstash.service to /etc/systemd/system/logstash.service.

beats 收集数据：

• filebeats 日志文件收集 并发送 数据
• Metricbeat 输送各种系统和服务统计数据，从 CPU 到内存，从 Redis 到 Nginx
• packetbeat 网络数据包

[root@izbp1enj8meljjp5l7yj7ez ~]# yum install filebeat

[root@izbp1enj8meljjp5l7yj7ez ~]# vim /etc/filebeat/filebeat.yml ####默认 filebeat 输出到 elasticsearch ; 注释掉；然后 改成 传输到 logstash （filebeat.outputs

#output.elasticsearch:

# Array of hosts to connect to.

#hosts: ["localhost:9200"]

#----------------------------- Logstash output --------------------------------

output.logstash:

#The Logstash hosts

hosts: ["localhost:5044"]

[root@izbp1enj8meljjp5l7yj7ez ~]# filebeat modules enable system ###启用系统模块

Enabled system

[root@izbp1enj8meljjp5l7yj7ez ~]# filebeat modules list

Enabled:

system

 

Disabled:

apache

auditd

haproxy

ibmmq

icinga

iis

iptables

kafka

kibana

logstash

misp

mongodb

mssql

mysql

[root@izbp1enj8meljjp5l7yj7ez modules.d]# ls /etc/filebeat/modules.d/

apache.yml.disabled elasticsearch.yml.disabled mysql.yml.disabled rabbitmq.yml.disabled kafka.yml.disabled redis.yml.disabled kibana.yml.disabled haproxy.yml.disabled logstash.yml.disabled nginx.yml.disabled system.yml mongodb.yml.disabled mssql.yml.disabled postgresql.yml.disabled

[root@izbp1enj8meljjp5l7yj7ez modules.d]# sudo filebeat setup --template -E output.logstash.enabled=false -E 'output.elasticsearch.hosts=["localhost:9200"]' ##加载索引模板

Flag --template has been deprecated, please use --index-management instead

Exiting: Couldn't connect to any of the configured Elasticsearch hosts. Errors: [Error connection to Elasticsearch http://localhost:9200: Get http://localhost:9200: dial tcp 127.0.0.1:9200: connect: connection refused]

[root@izbp1enj8meljjp5l7yj7ez ~]# mkdir /var/run/elasticsearch

[root@izbp1enj8meljjp5l7yj7ez ~]# chown -R elasticsearch:elasticsearch /var/run/elasticsearch

[root@izbp1enj8meljjp5l7yj7ez ~]# chown -R elasticsearch:elasticsearch /etc/elasticsearch/

[root@izbp1enj8meljjp5l7yj7ez ~]# systemctl start elasticsearch

Job for elasticsearch.service failed because the control process exited with error code. See "systemctl status elasticsearch.service" and "journalctl -xe" for details.

[root@izbp1enj8meljjp5l7yj7ez ~]# systemctl stop elasticsearch

[root@izbp1enj8meljjp5l7yj7ez ~]# systemctl start elasticsearch

Job for elasticsearch.service failed because the control process exited with error code. See "systemctl status elasticsearch.service" and "journalctl -xe" for details.

 

[root@izbp1enj8meljjp5l7yj7ez modules.d]# sudo filebeat setup -e -E output.logstash.enabled=false -E 'output.elasticsearch.hosts=["localhost:9200"]' -E setup.kibana.host=localhost:5601 ##加载索引模板

 

==================================================================

[root@iz8vb4uwcxp5otbzjz54m3z ~]# cat /etc/logstash/conf.d/nginx_log.conf

input {

file {

path => ["/var/log/nginx/access.log"]

start_position => "beginning"

type => "nginx-access"

}

file {

path => ["/var/log/nginx/error.log"]

start_position => "beginning"

type => "nginx-error"

}

 

}

filter {

if [type] == "nginx-access"{

grok {

match => { "message" => "%{COMBINEDAPACHELOG} %{QS:x_forwarded_for}"}

}

}

if [type] == "nginx-error"{

grok {

match => { "message" => "\[(?<timestamp>%{YEAR}[./-]%{MONTHNUM}[./-]%{MONTHDAY}[- ]%{TIME})\] \[%{LOGLEVEL:severity}\] %{POSINT:pid}#%{NUMBER}: (?:, client: (?<clientip>%{IP}|%{HOSTNAME}))(?:, server: %{IPORHOST:server}?)(?:, request: %{QS:request})?(?:, upstream: (?<upstream>\"%{URI}\"|%{QS}))?(?:, host: %{QS:request_host})?(?:, referrer: \"%{URI:referrer}\")?" }

}

}

geoip {

source => "clientip"

}

useragent {

source => "agent"

target => "useragent"

}

date {

match => ["timestamp", "dd/MMM/yyyy:HH:mm:ss Z"]

}

}

output {

if [type] == "nginx-access" {

elasticsearch {

hosts => ["127.0.0.1:9200"] index => "logstash-nginx-access-%{+YYYY.MM.dd}"

}

}

if [type] == "nginx-error" {

elasticsearch {

hosts => ["127.0.0.1:9200"] index => "logstash-nginx-error-%{+YYYY.MM}"

}

}

stdout {

codec => rubydebug

}

}

检查配置：

[root@iz8vb4uwcxp5otbzjz54m3z ~]# /usr/share/logstash/bin/logstash --config.test_and_exit -f /etc/logstash/conf.d/nginx_log.conf

WARNING: Could not find logstash.yml which is typically located in $LS_HOME/config or /etc/logstash. You can specify the path using --path.settings. Continuing using the defaults

Could not find log4j2 configuration at path /usr/share/logstash/config/log4j2.properties. Using default config which logs errors to the console

[INFO ] 2020-10-27 15:24:28.323 [main] runner - Starting Logstash {"logstash.version"=>"7.9.3", "jruby.version"=>"jruby 9.2.13.0 (2.5.7) 2020-08-03 9a89c94bcc Java HotSpot(TM) 64-Bit Server VM 25.231-b11 on 1.8.0_231-b11 +indy +jit [linux-x86_64]"}

[WARN ] 2020-10-27 15:24:28.973 [LogStash::Runner] multilocal - Ignoring the 'pipelines.yml' file because modules or command line options are specified

[INFO ] 2020-10-27 15:24:31.612 [LogStash::Runner] Reflections - Reflections took 77 ms to scan 1 urls, producing 22 keys and 45 values

Configuration OK

[INFO ] 2020-10-27 15:24:33.983 [LogStash::Runner] runner - Using config.test_and_exit mode. Config Validation Result: OK. Exiting Logstash

测试收集和解析日志

如果Logstash当前正在运行，先停掉

systemctl stop logstash

指定配置文件执行，测试终端输出结果

/usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/nginx_log.conf

访问：

http://192.168.143.233:5601/app/kibana#/index_pattern