EFK

一、EFK是什么？

是一个实时的、分布式的可扩展的搜索引擎，允许进行全文、结构化搜索，它通常用于索引和搜索大量日志数据，也可用于搜索许多不同类型的文档。

Beats 是数据采集的得力工具。将 Beats 和您的容器一起置于服务器上，或者将 Beats 作为函数加以部署，然后便可在 Elastisearch 中集中处理数据。如果需要更加强大的处理性能，Beats 还能将数据输送到 Logstash 进行转换和解析。

Kibana 核心产品搭载了一批经典功能：柱状图、线状图、饼图、旭日图，等等。不仅如此，您还可以使用 Vega 语法来设计独属于您自己的可视化图形。所有这些都利用 Elasticsearch 的完整聚合功能。

Elasticsearch 通常与 Kibana 一起部署，Kibana 是 Elasticsearch 的一个功能强大的数据可视化 Dashboard，Kibana 允许你通过 web 界面来浏览 Elasticsearch 日志数据。

二、开始安装配置

#首先得有三台虚拟机
1：elasticsearch+jdk+logstash+kibana+redis

2:
elasticsearch+filebeat+jdk+apache

3:
elasticsearch+filebeat+jdk+apache
注：每一台虚拟机安装得软件包

三台都需要操作得

rpm -ivh jdk-8u131-linux-x64_.rpm
java -version 

#安装配置elasticsearch
rpm -ivh elasticsearch-6.6.2.rpm
vim /etc/elasticsearch/elasticsearch.yml
#在这个配置目录下修改配置文件
cat /etc/elasticsearch/elasticsearch.yml |grep -v "^#"
    cluster.name: cc
    path.data: /var/lib/elasticsearch
    path.logs: /var/log/elasticsearch
    network.host: 192.168.157.137
    http.port: 9200
    discovery.zen.ping.unicast.hosts:  ["192.168.157.137", "192.168.157.138","192.168.157.139"]

systemctl enable elasticsearch
systemctl start elasticsearch
systemctl sratus elasticsearch

tailf /var/log/elasticsearch/cc.log

#配置完文件和启动完之后查看一下端口号是否起来
netstat -lptnu |grep 9200

137服务器上操作

rpm -ivh kibana-6.6.2-x86_64.rpm
vi /etc/kibana/kibana.yml

cat kibana.yml |grep -v '^#'|sed '/^$/d'
    server.port: 5601
	server.host: "192.168.157.137"
	elasticsearch.hosts: ["http://192.168.157.137:9200"]

systemctl start kibana

netstat  -lptnu|grep 5601

138服务器上操作

#安装配置logstash和redis
rpm -ivh logstash-6.6.0.rpm
vi /etc/logstash/conf.d/httpd.conf

#编辑文件

#编辑文件
  input {
      redis {
              data_type => "list"
              host => "192.168.157.138"
              password => "111111"
              port => "6379"
              db => "1"
              key => "filebeat-httpd"
  	    }
  }
  output {
  	elasticsearch {
              hosts => ["192.168.190.167:9200"]
              index => "redis-httpdlog-%{+YYYY.MM.dd}"
      	}
  }


systemctl start logstash
chmod 777 /var/log -R
tailf /var/log/elasticsearch/cc.log

#安装部署redis

yum -y install gcc gcc-c++ pcre-devel zlib-devel

tar zxf redis-5.0.0.tar.gz

cp -r redis-5.0.0 /usr/local/redis

cd /usr/local/redis

make && make install

ln -s /usr/local/redis/src/redis-server /usr/bin/redis-server

ln -s /usr/local/redis/src/redis-cli /usr/bin/redis-cli

vi /usr/local/redis/redis.conf

#修改配置文件
cat /usr/local/redis/redis.conf |grep -v "^#"
  bind 192.168.157.138
  requirepass 111111
  
  redis-server /usr/local/redis/redis.conf
  echo 511 > /proc/sys/net/core/somaxconn
  echo "vm.overcommit_memory = 1" >> /etc/sysctl.conf
  echo "echo never > /sys/kernel/mm/transparent_hugepage/enabled" >> /etc/

vim /usr/local/redis/redis.conf
  136  no--->yes

redis-server /usr/local/redis/redis.conf

139服务器上操作

#安装filebeat

yum -y install filebeat-6.8.1-x86_64.rpm

#修改配置文件
vim /etc/filebeat/filebeat.yml

filebeat.inputs:
		- type: log
  			enabled: true
  		paths:
    	- /var/log/httpd/access_log
		filebeat.config.modules:
  			path: ${path.config}/modules.d/*.yml
  			reload.enabled: false
  			setup.ilm.enabled: false
  			setup.template.name: "filebeat-httpd"
  			setup.template.pattern: "filebeat-httpd-*"
			setup.template.settings:
  			index.number_of_shards: 3
		setup.kibana:
	output.redis:
 		hosts: ["192.168.157.137:6379"]
  		key: "filebeat-httpd" 
  		db: 1 
  		timeout: 5 
  		password: 111111
	processors:
  		- add_host_metadata: ~
  		- add_cloud_metadata: ~

#测试redis收集httpd日志
redis-cli -h 192.168.157.137
 	192.168.157.137:6379> auth 111111
 	192.168.157.137:6379> get *
 	(nil)
 	192.168.157.137:6379> KEYS *
 	(empty list or set)
 	192.168.157.137:6379> SELECT 1
 	OK
 	192.168.157.137:6379[1]> KEYS *
 	"filebeat-httpd"

#浏览器打开192.168.190.167:5601测试