ELK 环境搭建总结

开始动手前的说明

我搭建这一套环境的时候是基于docker搭建的，用到了docker-compose，所以开始前要先安装好docker 、 docker-compose，并简单的了解docker 、 docker-compose的用法。

前言

ELK 是什么？
ELK 指：ElasticSearch + Logstash + Kibana

ELK 用来干什么？
ELK 可以用来收集日志并进行日志分析，实现日志的统一管理，帮助开发人员和运维人员快速分析日志，快速发现问题。

当然它还有很多非常多实用功能，需要您去自行挖掘。

这里使用Filebeat进行日志收集并将收集上来的日志发送给ELK。

es:

Elasticsearch 是一个分布式、RESTful 风格的搜索和数据分析引擎。

kibana:

Kibana 是通向 Elastic 产品集的窗口。 它可以在 Elasticsearch 中对数据进行视觉探索和实时分析。

logstash:

Logstash 是开源的服务器端数据处理管道，能够同时从多个来源采集数据，转换数据，然后将数据发送到您最喜欢的“存储库”中

filebeat:

轻量级收集日志的服务，并且可以将收集的日志发送给 es、logstash、kafka、redis

filebeat 概览图
[外链图片转存失败,源站可能有防盗链机制,建议将图片保存下来直接上传(img-sT9F2wH1-1585614949939)(https://www.elastic.co/guide/en/beats/filebeat/current/images/filebeat.png)]

ELK日志数据收集时序图

接下来开始动手操作。

准备工作：

$ mkdir ELK_pro
$ cd ELK_pro
$ touch docker-compose.yml
$ touch Dockerfile
$ touch filebeat.yml
$ touch kibana.yml
$ touch logstash-pipeline.conf
$ touch logstash.yml

ElasticSearch 环境搭建

我是参考官网的例子直接写的docker-compose.yml，然后做了小的改动。下面是我改动之后的配置：

version: "3"

services:
  es01:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.6.1
    container_name: es01
    environment:
      - node.name=es01
      - cluster.name=es-docker-cluster
      - discovery.seed_hosts=es02,es03
      - cluster.initial_master_nodes=es01,es02,es03
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
      - xpack.security.enabled=true
      - xpack.security.authc.accept_default_password=true
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/certificates/elastic-certificates.p12
      - xpack.security.transport.ssl.truststore.path=/usr/share/elasticsearch/config/certificates/elastic-certificates.p12
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - data01:/usr/share/elasticsearch/data
      - ./elastic-certificates.p12:/usr/share/elasticsearch/config/certificates/elastic-certificates.p12
    ports:
      - 9200:9200
    networks:
      - falling_wind
  es02:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.6.1
    container_name: es02
    environment:
      - node.name=es02
      - cluster.name=es-docker-cluster
      - discovery.seed_hosts=es01,es03
      - cluster.initial_master_nodes=es01,es02,es03
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
      - xpack.security.enabled=true
      - xpack.security.authc.accept_default_password=true
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/certificates/elastic-certificates.p12
      - xpack.security.transport.ssl.truststore.path=/usr/share/elasticsearch/config/certificates/elastic-certificates.p12
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - data02:/usr/share/elasticsearch/data
      - ./elastic-certificates.p12:/usr/share/elasticsearch/config/certificates/elastic-certificates.p12
    networks:
      - falling_wind

  es03:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.6.1
    container_name: es03
    environment:
      - node.name=es03
      - cluster.name=es-docker-cluster
      - discovery.seed_hosts=es01,es02
      - cluster.initial_master_nodes=es01,es02,es03
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
      - xpack.security.enabled=true
      - xpack.security.authc.accept_default_password=true
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/certificates/elastic-certificates.p12
      - xpack.security.transport.ssl.truststore.path=/usr/share/elasticsearch/config/certificates/elastic-certificates.p12
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - data03:/usr/share/elasticsearch/data
      - ./elastic-certificates.p12:/usr/share/elasticsearch/config/certificates/elastic-certificates.p12
    networks:
      - falling_wind
volumes:
  data01:
    driver: local
  data02:
    driver: local
  data03:
    driver: local

networks:
  falling_wind:
    driver: bridge

这个配置我是加了证书认证的。

下面请看证书生成方法：

1. 进入docker (es01)：

$ docker ps
$ docker exec -it 容器ID或名称 /bin/sh

2. 生成证书并copy

$ cd bin
$ elasticsearch-certutil ca
$ elasticsearch-certutil cert --ca elastic-stack-ca.p12
$ exit
$ docker cp 容器ID:/usr/share/elasticsearch/elastic-certificates.p12 .

# 注意：最后的点不要忘记了。

3. 设置es01的密码：

$ docker ps
$ docker exec -it 容器ID或名称 /bin/sh
$ cd bin
$ elasticsearch-setup-passwords interactive

# 按照提示设置密码即可

kibana 环境搭建

配置 kibana

docker-compose.yml

  kibana:
    image: docker.elastic.co/kibana/kibana:7.6.1
    container_name: kibana_7_61
    ports:
      - "5601:5601"
    volumes:
      - ./kibana.yml:/usr/share/kibana/config/kibana.yml
    networks:
      - falling_wind
    depends_on:
      - es01

kibana.yml

server.name: kibana
server.host: "0"
elasticsearch.hosts: ["http://172.18.114.219:9200"]
xpack.monitoring.ui.container.elasticsearch.enabled: true
elasticsearch.username: your username
elasticsearch.password: your password

logsstash 环境搭建

配置logsstash

docker-compose.yml

  logstash:
    image: docker.elastic.co/logstash/logstash:7.6.1
    container_name: logstash_7_61
    ports: 
      - "5044:5044"
    volumes: 
      - ./logstash.yml:/usr/share/logstash/config/logstash.yml
      - ./logstash-pipeline.conf:/usr/share/logstash/conf.d/logstash-pipeline.conf
    networks:
      - falling_wind

logstash.yml

path.config: /usr/share/logstash/conf.d/*.conf
path.logs: /var/log/logstash

logstash-pipeline.conf

input {
  beats {
    port => 5044
    codec => json
  }
  tcp {
    port => 8000
    codec => json
  }
}

output {
  elasticsearch {
    hosts => ["172.18.114.219:9200"]
    index => "falling-wind"
    user => "your username"
    password => "your password"
  }
  stdout {
    codec => rubydebug
  }
}

filebeat 环境搭建

配置 filebeat

docker-compose.yml

  filebeat:
    container_name: filebeat_7_61
    build:
      context: .
      dockerfile: Dockerfile
    volumes: 
      - /var/logs:/usr/share/filebeat/logs
    networks:
      - falling_wind

Dockerfile

FROM docker.elastic.co/beats/filebeat:7.6.1
COPY filebeat.yml /usr/share/filebeat/filebeat.yml
USER root
RUN chown root:filebeat /usr/share/filebeat/filebeat.yml
RUN chown root:filebeat /usr/share/filebeat/data/meta.json

说明：官网上的Dockerfile最后还加了 USER filebeat，按理说应该不会出现什么问题，但是启动总是会报权限不足：/usr/share/filebeat/data/meta.json，所以我暂时将这一句去掉就好了。

filebeat.yml

filebeat.inputs:
  - type: log
    paths:
      - /usr/share/filebeat/logs/falling-wind/*.log
    multiline.pattern: '^[[:space:]]'
    multiline.negate: false
    multiline.match: after
    tags: ["falling-wind"]
  - type: log
    paths:
      - /usr/share/filebeat/logs/celery/*.log
    multiline.pattern: '^[[:space:]]'
    multiline.negate: false
    multiline.match: after
    tags: ["celery"]
  - type: log
    paths:
      - /usr/share/filebeat/logs/gunicorn/*.log
    multiline.pattern: '^[[:space:]]'
    multiline.negate: false
    multiline.match: after
    tags: ["gunicorn"]
  - type: log
    paths:
      - /usr/share/filebeat/logs/supervisor/*.log
    tags: ["supervisor"]

#============================= Filebeat modules ===============================
filebeat.config.modules:
  # Glob pattern for configuration loading
  path: ${path.config}/modules.d/*.yml
  # Set to true to enable config reloading
  reload.enabled: true

output.logstash:
  hosts: ["172.18.114.219:5044"]

注意合并多行信息的配置：

将堆栈信息合并：

multiline.pattern: '^[[:space:]]'
multiline.negate: false
multiline.match: after

总结

一共需要的配置文件：

• docker-compose.yml
• Dockerfile： 构建filebeat镜像
• elastic-certificates.p12：证书文件
• filebeat.yml
• kibana.yml
• logstash-pipeline.conf
• logstash.yml

docker-compose.yml 完整版：

version: "3"

services:
  es01:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.6.1
    container_name: es01
    environment:
      - node.name=es01
      - cluster.name=es-docker-cluster
      - discovery.seed_hosts=es02,es03
      - cluster.initial_master_nodes=es01,es02,es03
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
      - xpack.security.enabled=true
      - xpack.security.authc.accept_default_password=true
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/certificates/elastic-certificates.p12
      - xpack.security.transport.ssl.truststore.path=/usr/share/elasticsearch/config/certificates/elastic-certificates.p12
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - data01:/usr/share/elasticsearch/data
      - ./elastic-certificates.p12:/usr/share/elasticsearch/config/certificates/elastic-certificates.p12
    ports:
      - 9200:9200
    networks:
      - falling_wind
  es02:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.6.1
    container_name: es02
    environment:
      - node.name=es02
      - cluster.name=es-docker-cluster
      - discovery.seed_hosts=es01,es03
      - cluster.initial_master_nodes=es01,es02,es03
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
      - xpack.security.enabled=true
      - xpack.security.authc.accept_default_password=true
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/certificates/elastic-certificates.p12
      - xpack.security.transport.ssl.truststore.path=/usr/share/elasticsearch/config/certificates/elastic-certificates.p12
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - data02:/usr/share/elasticsearch/data
      - ./elastic-certificates.p12:/usr/share/elasticsearch/config/certificates/elastic-certificates.p12
    networks:
      - falling_wind

  es03:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.6.1
    container_name: es03
    environment:
      - node.name=es03
      - cluster.name=es-docker-cluster
      - discovery.seed_hosts=es01,es02
      - cluster.initial_master_nodes=es01,es02,es03
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
      - xpack.security.enabled=true
      - xpack.security.authc.accept_default_password=true
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/certificates/elastic-certificates.p12
      - xpack.security.transport.ssl.truststore.path=/usr/share/elasticsearch/config/certificates/elastic-certificates.p12
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - data03:/usr/share/elasticsearch/data
      - ./elastic-certificates.p12:/usr/share/elasticsearch/config/certificates/elastic-certificates.p12
    networks:
      - falling_wind

  kibana:
    image: docker.elastic.co/kibana/kibana:7.6.1
    container_name: kibana_7_61
    ports:
      - "5601:5601"
    volumes:
      - ./kibana.yml:/usr/share/kibana/config/kibana.yml
    networks:
      - falling_wind
    depends_on:
      - es01

  logstash:
    image: docker.elastic.co/logstash/logstash:7.6.1
    container_name: logstash_7_61
    ports: 
      - "5044:5044"
    volumes: 
      - ./logstash.yml:/usr/share/logstash/config/logstash.yml
      - ./logstash-pipeline.conf:/usr/share/logstash/conf.d/logstash-pipeline.conf
    networks:
      - falling_wind

  filebeat:
    container_name: filebeat_7_61
    build:
      context: .
      dockerfile: Dockerfile
    volumes: 
      - /var/logs:/usr/share/filebeat/logs
    networks:
      - falling_wind

volumes:
  data01:
    driver: local
  data02:
    driver: local
  data03:
    driver: local

networks:
  falling_wind:
    driver: bridge

Enjoy your code!