docker-compose搭建ELK+filebeat

目录

一、配置文件路径

二、elk-docker-compose.yml

三、kibana.yml

四、logstash

4.1、logstash.conf

4.2、logstash.yml

4.3、patterns

五、filebeat.yml

---

一、配置文件路径

kibana.yml、logstash.conf、logstash.yml、java、filebeat.yml配置文件请防止在/home/conf/elk路径下

二、elk-docker-compose.yml

# docker-compose -f elk-docker-compose.yml up -d
version: "3"  #版本号
services:
  elasticsearch:  #服务名称（不是容器名）
    image: elasticsearch:7.6.2  #使用的镜像
    ports:
      - "9200:9200"     #暴露的端口信息和docker run -d -p 80:80一样
    restart: "always"   #重启策略，能够使服务器始终运行，生产环境推荐使用
    environment:
      - discovery.type=single-node
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
    container_name: elasticsearch  #容器名称
  kibana:  
    image: kibana:7.6.2  
    ports:
      - "5601:5601"   
    restart: "always"   
    container_name: kibana   
    volumes:
      - /home/conf/elk/kibana.yml:/etc/kibana/kibana.yml
    links:
      - elasticsearch:elasticsearch  #容器关联elasticsearch是别名
  logstash: 
    image: logstash:7.6.2  
    restart: "always"    
    container_name: logstash  
    ports:
      - "5044:5044"
      - "5045:5045"
    volumes:
      - /home/conf/elk/logstash.conf:/usr/share/logstash/pipeline/logstash.conf:rw
      - /home/conf/elk/logstash.yml:/usr/share/logstash/config/logstash.yml:rw
      - /home/conf/elk/java:/usr/share/logstash/patterns/java:rw
    links:
      - elasticsearch:elasticsearch
  filebeat:
    image: elastic/filebeat:7.6.2
    restart: "always"
    container_name: filebeat
    volumes:
      - /home/conf/elk/filebeat.yml:/usr/share/filebeat/filebeat.yml
      - /tools/logs/:/tools/logs/
    user: root   
    links:
      - logstash:logstash 

三、kibana.yml

elasticsearch.url: "http://elasticsearch:9200"
server.host: "0.0.0.0"

四、logstash

4.1、logstash.conf

input {
  beats {
    port => 5044
  }
}

filter {
  if [event][module] == "nginx" {
    mutate { add_field => { "[@metadata][target_index]" => "jira-nginx-%{+YYYY.MM}"} }
  }else if [fields][source] == "itsp" {
    mutate { add_field => { "[@metadata][target_index]" => "itsp-%{+YYYY.MM.dd}" } }
  }else if [fields][logfrom] == "boot-zipkin" {
    grok {
      patterns_dir => ["/usr/share/logstash/patterns"]
      match => { "message" => "%{DATE_CN:timestamp}\|%{LOGLEVEL:level}\|%{POSINT:pid}\|%{DATA:thread}\|%{DATA:appname}\|%{DATA:traceId}\|%{DATA:spanId}\|%{DATA:spanExport}\|%{DATA:class}\|%{JAVALOGMESSAGE:m
sg}" }
     remove_field => ["message"]
    }

    date {
      timezone => "Asia/Chongqing"
      match => ["timestamp", "yyyy-MM-dd HH:mm:ss.SSS"]
      target => "@timestamp"
      remove_field => "timestamp"
    }
    mutate {
        add_field => { "[@metadata][target_index]" => "boot-zipkin-%{+YYYY.MM}"}
        rename => {"msg" => "message"}
     }
    ruby{
        code => "event.set('date2',(event.get('@timestamp').to_f.round(3)*1000).to_i)"
    }

  }
}

output {
  elasticsearch {
    hosts => ["http://elasticsearch:9200"]
    index => "%{[@metadata][target_index]}"
    #user => "elastic"
    #password => "Tpjkyyxtsb2020"
  }
}

该路径为下述patterns定义的路径

4.2、logstash.yml

暂时未配置，可以创建个空文件

4.3、patterns

vim java

#日期正则自定义
DATE_CN %{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{HOUR}:%{MINUTE}:%{SECOND}

五、filebeat.yml

filebeat.inputs:

- type: log
  enabled: true
  paths:
    - /tools/logs/*.log
  fields:
    logfrom: boot-zipkin
  multiline.pattern: '^[[:space:]]|^Caused by:'
  multiline.negate: false
  multiline.match: after

output.logstash:
  hosts: ["logstash:5044"]

processors:
  - add_host_metadata: ~
  - add_cloud_metadata: ~
  - add_docker_metadata: ~
  - add_kubernetes_metadata: ~

 /tools/logs/*.log为项目日志输出位置