ELK进阶详解

实验环境

192.168.170.8  node1 kibana
192.168.170.9  node2 els
192.168.170.10 node3 logstash1
192.168.170.11 node4 redis
192.168.170.12 node5 logstash2
192.168.170.22 node6 haproxy+web

保证所有主机的主机名都能被解析,同时保证ntp时间是同步的,同时关闭selinux和防火墙功能。

安装nginx服务

安装开发环境所依赖的包

[root@node6 ~]# yum -y install gcc automake autoconf libtool make openssl openssl-devel gcc gcc-c++ pcre pcre-devel zlib zlib-devel 
	nginx的Rewrite模块和HTTP核心模块会使用到PCRE正则表达式语法: pcre和pcre-devel 
	nginx的各种模块中需要使用gzip压缩: zlib zlib-devel
	openssl是一个安全套接字层密码库,nginx要支持https,需要使用openssl

下载nginx包并解压(到/usr/local/src目录中),编译安装

[root@node6 ~]# cd /usr/local/src
[root@node6 src]# wget http://nginx.org/download/nginx-1.12.2.tar.gz
[root@node6 src]# tar xvf nginx-1.12.2.tar.gz
[root@node6 src]# cd nginx-1.12.2
[root@node6 nginx-1.12.2]# ./configure
--prefix=/usr/local/nginx/ \
--user=nginx \
--group=nginx \
--with-http_stub_status_module \
--with-http_ssl_module \
--with-http_sub_module \
--with-http_realip_module \
--with-http_image_filter_module
[root@node6 nginx-1.12.2]# make && make install
[root@node6 nginx-1.12.2]# nginx -V

设置nginx为系统服务

[root@node6 ~]# vim /usr/lib/systemd/system/nginx.service
[Unit]
Description=nginx - high performance web server
Documentation=http://nginx.org/en/docs/
After=network-online.target remote-fs.target nss-lookup.target
Wants=network-online.target
 
[Service]
Type=forking
PIDFile=/var/run/nginx.pid
ExecStartPre=/usr/sbin/nginx -t -c /etc/nginx/nginx.conf
ExecStart=/usr/sbin/nginx -c /etc/nginx/nginx.conf
ExecReload=/bin/kill -s HUP $MAINPID
ExecStop=/bin/kill -s TERM $MAINPID
 
[Install]
WantedBy=multi-user.target

配置web服务

[root@node6 conf.d]# vi vhost.conf
server {
        listen 80;
        server_name www.node6.com;
        root /data/nginx/html;
}
[root@node6 conf.d]# mkdir -pv /data/nginx/html/
[root@node6 conf.d]# cd /data/nginx/html/
[root@node6 html]#
[root@node6 html]# vi index.html
<h1>Test Page</h1>

 启动服务,设置开机启动

[root@node6 ~]# systemctl start nginx
[root@node6 ~]# systemctl enable nginx
[root@node6 ~]# ss -tunlp | grep 80

测试web页面访问:

[root@node6 ~]# curl http://192.168.170.22
Test Page

nginx日志转json格式

编辑nginx配置文件将nginx转json文件写在http段中,
[root@node6 ~]# vi /etc/nginx/nginx.conf
	log_format  json_log '{"@timestamp":"$time_iso8601",'
	'"host":"$server_addr",'
	'"clientip":"$remote_addr",'
	'"size":$body_bytes_sent,'
	'"responsetime":$request_time,'
	'"upstreamtime":"$upstream_response_time",'
	'"upstreamhost":"$upstream_addr",'
	'"http_host":"$host",'
	'"url":"$uri",'
	'"domain":"$host",'
	'"xff":"$http_x_forwarded_for",'
	'"referer":"$http_referer",'
	'"status":"$status"}';
     access_log /var/log/nginx/access.log json_log;

重读配置文件:

[root@node6 ~]# nginx -t
 nginx: the configuration file /usr/local/nginx//conf/nginx.conf syntax is ok
 nginx: configuration file /usr/local/nginx//conf/nginx.conf test is successful
[root@node6 ~]#systemctl restart nginx

验证nginx日志转json日志是否正确,通过浏览器json校验工具

[root@node6 ~]# tail /var/log/nginx/access.log  -f
172.17.1.112 - - [15/Apr/2019:19:13:23 +0800] "GET /favicon.ico HTTP/1.1" 404 169 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:66.0) Gecko/20100101 Firefox/66.0" "-"
172.17.1.112 - - [15/Apr/2019:19:23:47 +0800] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:66.0) Gecko/20100101 Firefox/66.0" "-"
{"@timestamp":"2019-04-15T19:23:47+08:00","host":"192.168.170.22","clientip":"172.17.1.112","size":0,"responsetime":0.000,"upstreamtime":"-","upstreamhost":"-","http_host":"192.168.170.22","url":"/index.html","domain":"192.168.170.22","xff":"-","referer":"-","status":"304"}

校验完成后的文件信息如下:会提示是"正确的json"。

{
	"@timestamp": "2019-04-15T19:23:47+08:00",
	"host": "192.168.170.22",
	"clientip": "172.17.1.112",
	"size": 0,
	"responsetime": 0.000,
	"upstreamtime": "-",
	"upstreamhost": "-",
	"http_host": "192.168.170.22",
	"url": "/index.html",
	"domain": "192.168.170.22",
	"xff": "-",
	"referer": "-",
	"status": "304"
}

nginx上安装filebeat

官方文档:
https://www.elastic.co/guide/en/beats/filebeat/current/index.html
下载地址:
https://www.elastic.co/downloads/beats/filebeat
下载rpm包安装:

[root@node6 ~]# cd /usr/local/src/
[root@node6 ~]# wget https://www.elastic.co/downloads/beats/filebeat-5.4.3-x86_64.rpm
[root@node6 ~]# yum install filebeat-5.4.3-x86_64.rpm

配置filebeat收集系统日志并写入到redis;

[root@node6 filebeat]# grep -v "#" /etc/filebeat/filebeat.yml | grep -v "^$"
filebeat.prospectors:
- input_type: log 
  paths:
    - /var/log/messages
  exclude_lines: ["^DBG","^$"]
  document_type: system-log-0022
output.redis:
  enabled: true
  hosts: ["192.168.170.11:6379"]
  key: "system-log-0022"
  db: 10
  timeout: 5
  password: 123456

重启filebeat服务,查看状态

[root@node6 filebeat]# systemctl restart filebeat
[root@node6 filebeat]# systemctl status filebeat

验证filebeat启动后,是否与redis服务建立连接

[root@node4 ~]# lsof -n  -i:6379
COMMAND     PID USER   FD   TYPE DEVICE SIZE/OFF NODE NAME
redis-ser 13578 root    6u  IPv4  73120      0t0  TCP *:6379 (LISTEN)
redis-ser 13578 root   10u  IPv4 175877      0t0  TCP 192.168.170.11:6379->192.168.170.22:46586 (ESTABLISHED)

验证redis是否有数据,显示正常。 

[root@node4 ~]# redis-cli -h 192.168.170.11 -a 123456
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
192.168.170.11:6379> SELECT 10
OK
192.168.170.11:6379[10]> KEYS *
1) "system-log-0022"
192.168.170.11:6379[10]> 
192.168.170.11:6379[10]> LLEN system-log-0022
(integer) 2250

redis日志输出到elasticsearch中

[root@node3 conf.d]# vi filebeats.conf
input {
    redis {
        data_type => "list"
        host => "192.168.170.11"
        port => "6379"
        key => "system-log-0022"
        db => "10"
        password => "123456"
    }
}
output {
    if [type] == "system-log-0022" {
        elasticsearch {
            hosts => ["192.168.170.9:9200"]
            index => "system-log-0022-%{+YYYY.MM.dd}"
        }
    }
}

 测试语法格式并重启服务:

[root@node3 conf.d]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/filebeats.conf t
[root@node3 conf.d]# systemctl restart logstash

启动服务在通过后台日志监测启动状态

[root@node3 ~]# tail -f /var/log/logstash/logstash-plain.log

在redis-server中看到日志已经输出至logstash1

[root@node4 ~]# redis-cli -h 192.168.170.11 -a 123456
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
192.168.170.11:6379[10]> LLEN system-log-0022
(integer) 0
192.168.170.11:6379[10]> 

浏览器输入http://192.168.170.9:9100/可以看到日志已经到elasticsearch中
将日志添加到kibana中

(二)

filebeat收集ngix访间和系统日志:
配置 filebeat收集nginx日志文件传给redis

[root@node6 filebeat]# grep -v "#" /etc/filebeat/filebeat.yml | grep -v "^$"
filebeat.prospectors:
- input_type: log 
  paths:
    - /var/log/messages
  exclude_lines: ["^DBG","^$"]
  document_type: system-log-0022
- input_type: log 
  paths:
    - /var/log/nginx/access.log
  document_type: nginx-accesslog-0022
output.logstash:
  hosts: ["192.168.170.12:5044","192.168.170.12:5045"]
  enabled: true
  worker: 2
  compression_level: 3
  loadbalance: true
output.redis:
  enabled: true
  hosts: ["192.168.170.11:6379"]
  key: "system-log-0022"
  db: 10
  timeout: 5
  password: 123456
重启filebeat服务,查看启动状态是否正常
[root@node6 filebeat]# systemcel restart filebeat
[root@node6 filebeat]# systemcel status filebeat

配置logstash测试标准输出:

[root@node5 conf.d]# vi beats.conf
input {
	beats {
		port => 5044
		codec => "json"
	}
	beats {
		port => 5045
		codec => "json"
	}
}
output {
	stdout {
		codec => "rubydebug"
	}
}

验证logstash标准输入

[root@node5 conf.d]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/beats.conf -t
[root@node5 conf.d]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/beats.conf
[INFO ] 2019-04-15 22:49:04.455 [[main]<beats] Server - Starting server on port: 5044
[INFO ] 2019-04-15 22:49:04.458 [[main]<beats] Server - Starting server on port: 5045

在nginx服务器上通过curl 请求nginx页面

[root@node6 filebeat]# curl http://192.168.170.22
<h1>nginx test</h1>

在logstash2上输出nginx请求日志信息,说明logstash2标准输出正常。

[root@node5 ~]# tail -f /var/log/logstash/logstash-plain.log 
 at [Source: (String)"192.168.170.22 - - [15/Apr/2019:22:59:33 +0800] "GET / HTTP/1.1" 200 20 "-" "curl/7.29.0" "-""; line: 1, column: 9]>, :data=>"192.168.170.22 - - [15/Apr/2019:22:59:33 +0800] \"GET / HTTP/1.1\" 200 20 \"-\" \"curl/7.29.0\" \"-\""}

测试成功后,将 logstash2输出至 redis:

[root@node5 conf.d]#  vi /etc/logstash/conf.d/filebeats.conf
input {
	beats {
		port => 5044
		codec => "json"
	}
	beats {
		port => 5045
		codec => "json"
	}
}
output {
	if [type] == "system-log-0022" {
		redis {
			data_type => "list"
			host => "192.168.170.11"
			port => "6379"
			key => "system-log-0022"
			db => "15"
			password => "123456"
		}
	}
	if [type] == "nginx-accesslog-0022" {
		redis {
			data_type => "list"
			host => "192.168.170.11"
			port => "6379"
			key => "nginx-accesslog-0022"
			db => "15"
			password => "123456"
			codec => "json"
		}
	}
}

验证并重启动logstash服务

[root@node5 conf.d]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/filebeats.conf -t
[root@node5 conf.d]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/filebeats.conf 

重启filebeat服务,查看启动状态是否正常

[root@node6 filebeat]# systemcel restart filebeat
[root@node6 filebeat]# systemcel status filebeat

filebeat进程状态查看:

[root@node6 ~]# ps -ef | grep filebeat
root     27990     1  0 13:03 ?        00:00:01 /usr/share/filebeat/bin/filebeat -c /etc/filebeat/filebeat.yml -path.home /usr/share/filebeat -path.config /etc/filebeat -path.data /var/lib/filebeat -path.logs /var/log/filebeat
root     32590 27223  0 13:17 pts/2    00:00:00 grep --color=auto filebeat
[root@node6 ~]# 

filebeat启动netstart端口查看

[root@node6 ~]# netstat -anlp | grep filebeat
tcp        0      0 192.168.170.22:56230    192.168.170.12:5044     ESTABLISHED 27990/filebeat      
tcp        0      0 192.168.170.22:56232    192.168.170.12:5044     ESTABLISHED 27990/filebeat      
tcp        0      0 192.168.170.22:54962    192.168.170.12:5045     ESTABLISHED 27990/filebeat      
tcp        0      0 192.168.170.22:54964    192.168.170.12:5045     ESTABLISHED 27990/filebeat      
unix  3      [ ]         STREAM     CONNECTED     181992795 27990/filebeat  

Logstash 端查看是否连接到5044端口

[root@node5 ~]# lsof -n -i:5044
COMMAND  PID     USER   FD   TYPE DEVICE SIZE/OFF NODE NAME
java    4785 logstash  117u  IPv6  35973      0t0  TCP *:lxi-evntsvc (LISTEN)

在客户端压测nginx服务器

[root@node1 ~]# ab -n5000 -c100 http://192.168.170.22/index.html

访问nginx产生大量日志信息,验证redis是否有日志

[root@node4 ~]# redis-cli -h 192.168.170.11 -a 123456
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
192.168.170.11:6379> SELECT 15
OK
192.168.170.11:6379[15]> KEYS *
1) "system-log-0022"
2) "nginx-accesslog-0022"
192.168.170.11:6379[15]> 

查看redis连接正常

[root@node4 ~]# lsof -n -i:6379
COMMAND     PID USER   FD   TYPE DEVICE SIZE/OFF NODE NAME
redis-ser 13578 root    6u  IPv4  73120      0t0  TCP *:6379 (LISTEN)
redis-ser 13578 root    7u  IPv4 177645      0t0  TCP 192.168.170.11:6379->192.168.170.10:49892 (ESTABLISHED)
redis-ser 13578 root    8u  IPv4 178146      0t0  TCP 192.168.170.11:6379->192.168.170.22:54420 (ESTABLISHED)
redis-ser 13578 root   10u  IPv4 181518      0t0  TCP 192.168.170.11:6379->192.168.170.12:33714 (ESTABLISHED)
redis-ser 13578 root   11u  IPv4 181520      0t0  TCP 192.168.170.11:6379->192.168.170.12:33716 (ESTABLISHED)

配置另外一台logstash1服务从redis读取数据并写入到elasticsearch

[root@node3 conf.d]# vi /etc/logstash/conf.d/redis-els.conf
input {
	redis {
		data_type => "list"
		host => "192.168.170.11"
		port => "6379"
		key => "system-log-0022"
		db => "15"
		password => "123456"
	}
	redis {
		data_type => "list"
		host => "192.168.170.11"
		port => "6379"
		key => "nginx-accesslog-0022"
		db => "15"
		password => "123456"
		codec => "json"
	 }
}
output {
	if [type] == "system-log-0022" {
		elasticsearch {
		hosts => ["192.168.170.9:9200"]
		index => "system-log-0022-%{+YYYY.MM.dd}"
	    }
}
	if [type] == "nginx-accesslog-0022" {
		elasticsearch {
			hosts => ["192.168.170.9:9200"]
			index => "nginx-accesslog-0022-%{+YYYY.MM.dd}"
	 	}
	}
}

 检查语法并重启logstash服务

[root@node3 conf.d]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/redis-es.conf -t
[root@node3 conf.d]# systemctl restart logstash

验证elasticsearch是否有数据
浏览器输入http://192.168.170.9:9100/可以看到日志已经到elasticsearch中。

日志添加到kibana上
kibana验证nginx访问日志和系统访问日志。

  • 0
    点赞
  • 0
    收藏
    觉得还不错? 一键收藏
  • 0
    评论

“相关推荐”对你有帮助么?

  • 非常没帮助
  • 没帮助
  • 一般
  • 有帮助
  • 非常有帮助
提交
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值