1.ELK架构
1.1 具体基于 Filebeat 的 ELK 分布式集中日志解决方案架构如图所示
1.2 软件版本
ELK 的版本需要一致,不然可能导致 ELK 无法使用。filebeat 的版本不需要特别注意,无特殊需求与本文一致即可。
Kibana:7.1.1
Filebeat:7.0.1
Logstash:7.1.1
Elasticsearch:7.1.1
1.3 环境
docker-compose
网盘地址有所有的镜像和写好的yml
链接:https://pan.baidu.com/s/1SR9P21SuO6ZE9Mk93T-FNw?pwd=9z2t
提取码:9z2t
--来自百度网盘超级会员V1的分享
2.ELK编写yaml
version: "3"
services:
es-master:
container_name: es-master
image: elasticsearch:7.1.1
hostname: es-master
restart: always
user: root
ports:
- 9200:9200
- 9300:9300
volumes:
- ./elasticsearch/master/conf/es-master.yml:/usr/share/elasticsearch/config/elasticsearch.yml
- ./elasticsearch/master/data:/usr/share/elasticsearch/data
- ./elasticsearch/master/logs:/usr/share/elasticsearch/logs
environment:
- "ES_JAVA_OPTS=-Xms512m -Xmx512m"
- "TZ=Asia/Shanghai"
networks:
- lili_net
kibana:
container_name: kibana
hostname: kibana
image: kibana:7.1.1
restart: always
ports:
- 5601:5601
volumes:
- ./kibana/conf/kibana.yml:/usr/share/kibana/config/kibana.yml
environment:
- elasticsearch.hosts=http://es-master:9200
- "TZ=Asia/Shanghai"
networks:
- lili_net
logstash:
container_name: logstash
hostname: logstash
image: logstash:7.1.1
command: logstash -f ./conf/logstash-filebeat.conf
restart: always
volumes:
- ./logstash/conf/logstash-filebeat.conf:/usr/share/logstash/conf/logstash-filebeat.conf
environment:
- elasticsearch.hosts=http://es-master:9200
- xpack.monitoring.elasticsearch.hosts=http://es-master:9200
- "TZ=Asia/Shanghai"
ports:
- 5044:5044
networks:
- lili_net
#########################################
networks:
lili_net:
name: lili_net
driver: bridge
ipam:
config:
- subnet: "172.100.0.0/16"
2.1 编写对应的挂载目录yml
2.1.1 es-master.yml
# 集群名称
cluster.name: es-cluster
# 节点名称
node.name: es-master
# 是否可以成为master节点
node.master: true
# 是否允许该节点存储数据,默认开启
node.data: true
# 网络绑定
network.host: 0.0.0.0
# 设置对外服务的http端口
http.port: 9200
# 设置节点间交互的tcp端口
transport.port: 9300
# 集群发现
discovery.seed_hosts:
- es-master
# 手动指定可以成为 mater 的所有节点的 name 或者 ip,这些配置将会在第一次选举中进行计算
cluster.initial_master_nodes:
- es-master
# 支持跨域访问
http.cors.enabled: true
http.cors.allow-origin: "*"
# 安全认证
xpack.security.enabled: false
#http.cors.allow-headers: "Authorization"
2.1.2 kibana.yml
# 服务端口
server.port: 5601
# 服务IP
server.host: "0.0.0.0"
# ES
elasticsearch.hosts: ["http://es-master:9200"]
# 汉化
i18n.locale: "zh-CN"
2.1.3 logstash-filebeat.conf
input {
beats {
port => 5044
}
}
# 分析、过滤插件,可以多个
filter {
grok {
match => ["message", "%{TIMESTAMP_ISO8601:logdate}"]
}
date {
match => ["logdate", "yyyy-MM-dd HH:mm:ss.SSS"]
target => "@timestamp"
}
}
output {
elasticsearch {
hosts => "http://es-master:9200"
index => "%{[fields][log_topics]}-%{+YYYY.MM.dd}"
document_type => "%{[@metadata][type]}"
}
}
logstash的所有配置文件都在这个里面修改
3.启动成功
root@localhost(192.168.199.54)/data/ELK>docker-compose ps
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
es-master elasticsearch:7.1.1 "/usr/local/bin/dock…" es-master About an hour ago Up About an hour 0.0.0.0:9200->9200/tcp, :::9200->9200/tcp, 0.0.0.0:9300->9300/tcp, :::9300->9300/tcp
kibana kibana:7.1.1 "/usr/local/bin/kiba…" kibana About an hour ago Up About an hour 0.0.0.0:5601->5601/tcp, :::5601->5601/tcp
logstash logstash:7.1.1 "/usr/local/bin/dock…" logstash About an hour ago Up About an hour 0.0.0.0:5044->5044/tcp, :::5044->5044/tcp, 9600/tcp
root@localhost(192.168.199.54)/data/ELK>