centos7使用docker compose部署ELK

说明：1、一定要先不要配置那么多配置文件，去除掉一些，先让docker compose启动相关服务能访问的时候，使用拷贝方法，把相关的配置文件拷贝出来在外面修改，这样保险一些，不然容易配置文件错误无法启动问题

 2、作者测试ELK版本（7.6.2）是可以通过下面步骤配置成功，试过（8.5.3）的版本不行，具体原因没有具体研究，网上有人说是因为elastic版本太高不兼容低版本的centos7，作者测试使用的操作系统是centos7.6

1、准备下载相关镜像

docker pull logstash:7.6.2
docker pull kibana:7.6.2
docker pull elasticsearch:7.6.2
docker pull elastic/filebeat:7.6.2

2、创建相关文件夹

新建文件夹使用命令：mkdir /opt/docker_elk

在/opt/docker_elk/elasticsearch新建plugins和data文件夹

设置目录/opt/docker_elk/filebeat/logs和拷贝相关日志文件smartbuilding-service.log

 3、设置相关配置文件

/opt/docker_elk/logstash/logstash.conf

input {
    beats {
       port => 5044
    }
}

filter {
    grok {
        pattern_definitions => {
            "QUALIFIED" => "[a-zA-Z0-9$_.]+"
        }

        match => {
            "message" => "%{TIMESTAMP_ISO8601:logdate}%{SPACE}\[%{USERNAME:logthread}\]%{SPACE}%{WORD:loglevel}%{SPACE}%{QUALIFIED:logclass:text}%{SPACE}-%{SPACE}%{GREEDYDATA:logmsg:text}"

        }

    }
}


output {
    elasticsearch {
        hosts =>["elasticsearch:9200"]
        index => "cloud"
        template => "/etc/logstash/template.json"
        template_name => "logstash"
    }

}

/opt/docker_elk/logstash/template.json

{
  "template": "logstash-*",
  "settings": {
    "number_of_shards": 1,
    "number_of_replicas": 0
  },
  "mappings": {
    "properties": {
      "logclass": {
        "type": "text"
      },
      "logdate": {
        "type": "date",
        "format": "yyyy-MM-dd HH:mm:ss.SSS"
      },
      "loglevel": {
        "type": "keyword"
      },
      "logthread": {
        "type": "keyword"
      },
      "logmsg": {
        "type": "text"
      }
    }
  }
}

/opt/docker_elk/docker-compose.yml

version: '3.7'
services:
  elasticsearch:
    image: elasticsearch:7.6.2
    container_name: elasticsearch
    privileged: true
    user: root
    environment:
      #设置集群名称为elasticsearch
      - cluster.name=elasticsearch 
      #以单一节点模式启动
      - discovery.type=single-node 
      #设置使用jvm内存大小
      - ES_JAVA_OPTS=-Xms512m -Xmx512m 
    volumes:
      - /opt/docker_elk/elasticsearch/plugins:/usr/share/elasticsearch/plugins
      - /opt/docker_elk/elasticsearch/data:/usr/share/elasticsearch/data
    ports:
      - 9200:9200
      - 9300:9300

  logstash:
    image: logstash:7.6.2
    restart: always
    container_name: logstash
    volumes:
      - /opt/docker_elk/logstash/pipeline/logstash.conf:/usr/share/logstash/pipeline/logstash.conf
      - /opt/docker_elk/logstash/template.json:/etc/logstash/template.json
    ports:
      - "5044:5044"
      - "9600:9600"
    environment:
      LS_JAVA_OPTS: "-Xms512m -Xmx512m"
    depends_on:
      - elasticsearch


  filebeat:
    image: elastic/filebeat:7.6.2
    restart: always
    container_name: filebeat
    volumes:
      - /opt/docker_elk/filebeat/logs:/var/log/filebeat/logs
    depends_on:
      - elasticsearch
      - kibana

  kibana:
    image: kibana:7.6.2
    container_name: kibana
    ports:
        - 5601:5601
    privileged: true
    depends_on:
      - elasticsearch 
    environment:
      #设置访问elasticsearch的地址
      - elasticsearch_url=elasricsearch:9200 

4、启动docker compose

#启动
docker compose -f docker-compose.yml up
#停止
docker compose -f docker-compose.yml down

注意；启动的时候可能elasticsearch会报错权限不足，使用下面命令赋予权限

chmod 777 /opt/docker_elk/elasticsearch/data

5、拷贝已经启动好的docker服务配置文件出来

注意：相关的容器id，可使用docker ps 查看

docker cp 9fc815e4334c:/usr/share/elasticsearch/config/elasticsearch.yml /opt/docker_elk/elasticsearch/config

docker cp ce9c723fecfb:/usr/share/kibana/config/kibana.yml /opt/docker_elk/kibana/config

docker cp f30b56380f92:/usr/share/logstash/config/logstash.yml /opt/docker_elk/logstash/pipeline

6、编辑拷贝出来的配置文件elasticsearch.yml

cluster.name: "docker-cluster"
network.host: 0.0.0.0
#http.cors.enabled: true #跨域配置
#http.cors.allow-origin: "*"
xpack.security.enabled: true  #开启密码配置

7、修改docker compose文件的elasticsearch服务

  - /opt/docker_elk/elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml

重启docker 相关服务

8、进入docker容器，开启账号密码认证模式

#进入es容器设置账号密码登录
docker exec -it elasticsearch /bin/bash 
#执行设置：elastic、apm_system、kibana_system、logstash_system、beats_system、remote_monitoring_user共6个用户账号密码
./bin/elasticsearch-setup-passwords interactive -u 'http://192.168.0.132:9200'
#退出容器

 9、修改配置文件kibana.yml

/opt/docker_elk/kibana/config

#
# ** THIS IS AN AUTO-GENERATED FILE **
#

# Default Kibana configuration for docker target
i18n.locale: "zh-CN"  #汉化
server.name: kibana
server.host: "0"
elasticsearch.hosts: [ "http://elasticsearch:9200" ]
xpack.monitoring.ui.container.elasticsearch.enabled: true
#elasticsearch.username: "kibana_system" #注意不能用elastic超管账号登录
elasticsearch.username: "elastic" 
elasticsearch.password: "填写你设置的密码"

10、设置logstash.yml

/opt/docker_elk/logstash/pipeline

http.host: "0.0.0.0"
xpack.monitoring.elasticsearch.hosts: [ "http://elasticsearch:9200" ]
xpack.monitoring.elasticsearch.username: "elastic"
xpack.monitoring.elasticsearch.password: "你设置的密码"

修改/opt/docker_elk/logstash/logstash.conf文件增加用户和密码

input {
    beats {
       port => 5044
    }
}

filter {
    grok {
        pattern_definitions => {
            "QUALIFIED" => "[a-zA-Z0-9$_.]+"
        }

        match => {
            "message" => "%{TIMESTAMP_ISO8601:logdate}%{SPACE}\[%{USERNAME:logthread}\]%{SPACE}%{WORD:loglevel}%{SPACE}%{QUALIFIED:logclass:text}%{SPACE}-%{SPACE}%{GREEDYDATA:logmsg:text}"

        }

    }
}


output {
    elasticsearch {
        hosts =>["elasticsearch:9200"]
        index => "cloud"
        template => "/etc/logstash/template.json"
        template_name => "logstash"
        user => "elastic"
        password => "你设置的密码"
    }

}

11、重新配置docker compose配置文件

version: '3.7'
services:
  elasticsearch:
    image: elasticsearch:7.6.2
    container_name: elasticsearch
    privileged: true
    user: root
    environment:
      #设置集群名称为elasticsearch
      - cluster.name=elasticsearch 
      #以单一节点模式启动
      - discovery.type=single-node 
      #设置使用jvm内存大小
      - ES_JAVA_OPTS=-Xms512m -Xmx512m 
    volumes:
      - /opt/docker_elk/elasticsearch/plugins:/usr/share/elasticsearch/plugins
      - /opt/docker_elk/elasticsearch/data:/usr/share/elasticsearch/data
      - /opt/docker_elk/elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
    ports:
      - 9200:9200
      - 9300:9300

  logstash:
    image: logstash:7.6.2
    restart: always
    container_name: logstash
    volumes:
      - /opt/docker_elk/logstash/pipeline/logstash.conf:/usr/share/logstash/pipeline/logstash.conf
      - /opt/docker_elk/logstash/pipeline/logstash.yml:/usr/share/logstash/config/logstash.yml
      - /opt/docker_elk/logstash/template.json:/etc/logstash/template.json
    ports:
      - "5044:5044"
      - "9600:9600"
    environment:
      LS_JAVA_OPTS: "-Xms512m -Xmx512m"
    depends_on:
      - elasticsearch


  filebeat:
    image: elastic/filebeat:7.6.2
    restart: always
    container_name: filebeat
    volumes:
      - /opt/docker_elk/filebeat/filebeat.yml:/usr/share/filebeat/filebeat.yml
      - /opt/docker_elk/filebeat/logs:/var/log/filebeat/logs
    depends_on:
      - elasticsearch
      - kibana

  kibana:
    image: kibana:7.6.2
    container_name: kibana
    ports:
        - 5601:5601
    privileged: true
    depends_on:
      - elasticsearch 
    environment:
      #设置访问elasticsearch的地址
      - elasticsearch_url=elasricsearch:9200 
    volumes:
      - /opt/docker_elk/kibana/config/kibana.yml:/usr/share/kibana/config/kibana.yml

重启docker服务后，就可正常使用ELK进行设置了

12、进入kibana

http://192.168.0.132:5601

账号：elastic

密码：你设置的密码

 

查看日志

 其他问题：

启动后，可能会发现elasticsearch状态显示yellow

curl -XPUT "http://192.168.0.132:9200/_settings" -H 'Content-Type: application/json' -d'
{
    "index" : {
        "number_of_replicas" : 0
    }
}'