docker部署ELK日志系统+kafka

docker部署ELK日志系统+kafka

1.部署elasticsearch

#拉取镜像
docker pull elasticsearch:7.6.2

# 创建文件夹
mkdir -p /data/elk/es/config
# 创建 elasticsearch.yml 文件
vi /data/elk/es/config/elasticsearch.yml
------------------------写入---------------------------
http.port: 9200
network.host: 0.0.0.0 
cluster.name: "my-el" 
http.cors.enabled: true 
http.cors.allow-origin: "*" 
------------------------结束---------------------------
#启动镜像
docker run -itd --name es -p elasticsearch:7.6.2
# 赋值容器中的data、logs文件
docker cp 容器id:/usr/share/elasticsearch/data /data/elk/es
docker cp 容器id:/usr/share/elasticsearch/logs /data/elk/es
#赋予权限
chown -R 1000:1000 /data/elk/es

#启动镜像
docker run -itd -p 9200:9200 -p 9300:9300 --name es -e ES_JAVA_OPTS="-Xms512m -Xmx512m" -e "discovery.type=single-node" --restart=always -v /data/elk/es/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml -v /data/elk/es/data:/usr/share/elasticsearch/data -v /data/elk/es/logs:/usr/share/elasticsearch/logs elasticsearch:7.6.2

2.elasticsearch-head可视化工具（可选）

# 拉取镜像
docker pull  mobz/elasticsearch-head:5

#启动镜像
docker run -d --restart=always -p 9100:9100 --name elasticsearch-head mobz/elasticsearch-head:5

# 创建文件夹
mkdir -p /data/elk/elasticsearch-head
# 进入文件夹下
cd /data/elk/elasticsearch-head
# 拷贝 vendor.js 文件到宿主机上
docker cp elasticsearch-head:/usr/src/app/_site/vendor.js ./
# 修改 vendor.js 文件内容
sed -i '/contentType:/s/application\/x-www-form-urlencoded/application\/json;charset=UTF-8/' vendor.js

sed -i '/var inspectData = s.contentType/s/application\/x-www-form-urlencoded/application\/json;charset=UTF-8/' vendor.js

# 删除容器
docker rm -f elasticsearch-head
# 运行容器
docker run -d --restart=always -p 9100:9100 -v /data/elk/elasticsearch-head/vendor.js:/usr/src/app/_site/vendor.js --name elasticsearch-head mobz/elasticsearch-head:5

3.部署kibana

#拉取镜像,版本与elasticsearch一致，避免出现不兼容问题
docker pull kibana:7.6.2

#获取elasticsearch容器ip
docker inspect --format '{{ .NetworkSettings.IPAddress }}' es

# 创建挂载配置
mkdir -p /data/elk/kibana
vi /data/elk/kibana/kibana.yml

#配置内容
#Default Kibana configuration for docker target
server.name: kibana
server.host: "0"
elasticsearch.hosts: ["http://elasticsearch容器ip:9200"]
xpack.monitoring.ui.container.elasticsearch.enabled: true
i18n.locale: "zh-CN"
kibana.index: ".kibana"

#启动镜像
docker run -d --restart=always --log-driver json-file --log-opt max-size=100m --log-opt max-file=2 --name kibana -p 5601:5601 -v /data/elk/kibana/kibana.yml:/usr/share/kibana/config/kibana.yml kibana:7.6.2

4.部署kafka（kafka不是必须的，用来解决io读取日志丢失问题,先将日志推送至kafka logstash异步读取从而缓解服务器压力）

#拉取zookeeper镜像,因为kafka依赖zookeeper
docker pull wurstmeister/zookeeper

#启动zookeeper镜像
#-v /etc/localtime:/etc/localtime 与服务器使用一样的时间设置
docker run -d  --name zookeeper -p 2181:2181 -v /etc/localtime:/etc/localtime wurstmeister/zookeeper

#拉取kafka镜像
docker pull wurstmeister/kafka

#启动kafka镜像
docker run -d --name kafka -p 9092:9092 \
-e KAFKA_BROKER_ID=0 \
-e KAFKA_ZOOKEEPER_CONNECT=localhost:2181 \
-e KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://localhost:9092 \
-e KAFKA_LISTENERS=PLAINTEXT://0.0.0.0:9092 \
-v /etc/localtime:/etc/localtime \
wurstmeister/kafka

#进入容器
docker exec -it 容器id bash
#进入到bin目录
cd /opt/kafka/bin/
#创建topic测试
./kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic elk-log
./kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic elk-log
#查看topic
kafka-topics.sh --list --zookeeper localhost:2181
#删除topic
kafka-topics.sh --delete --zookeeper localhost -topic elk-log

#测试是否成功
#生产者,发送消息
./kafka-console-producer.sh --broker-list localhost:9092 --topic elk-log
#消费者,订阅消息
./kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic elk-log

5.部署logstash

#拉取镜像,跟前面两个保持版本一样
docker pull logstash:7.6.2

#创建挂载配置
mkdir -p /data/elk/logstash
vi /data/elk/logstash/logstash.yml

#配置内容
http.host: "0.0.0.0"
xpack.monitoring.elasticsearch.hosts: [ "http://elasticsearch容器ip:9200" ]
xpack.monitoring.elasticsearch.username: elastic
xpack.monitoring.elasticsearch.password: changeme
path.config: /data/docker/logstash/conf.d/*.conf
path.logs: /var/log/logstash

mkdir -p /data/elk/logstash/conf.d
vi /data/elk/logstash/conf.d/syslog.conf

#配置内容
input {
  kafka {
    bootstrap_servers => "localhost:9092"
    topics => ["system-provder-log"] # kafka主题
    codec => "json"
    auto_offset_reset => "earliest" #从最早的偏移量开始消费
    decorate_events => true    #此属性会将当前topic、offset、group、partition等信息也带到message中
    type => "system_log" #所有插件通用属性,尤其在input里面配置多个数据源时很有用
  }
}
# 解决与中国本地时间偏移8小时问题
filter {
    ruby {
                code => "event.set('timestamp', event.get('@timestamp').time.localtime + 8*60*60)"
        }

        # grok 从日志正则匹配得到
        grok {
                match => {"message"=> "%{TIMESTAMP_ISO8601:timestamp}"}
        }

        mutate {
                convert => ["timestamp", "string"]
                gsub => ["timestamp", "T([\S\s]*?)Z", ""]
                gsub => ["timestamp", "-", "."]
        }
}
output {
  # 如果不需要打印可以直接删除
  stdout {
        codec => rubydebug {metadata => true}  #logstash控制台输出日志和@metadata信息
    }
 
  # 通过type用于区分不同来源的日志
  if [type] == "system_log" {
        elasticsearch {
            hosts => ["http://localhost:9200"]
            index => "system_log-%{timestamp}"
        }
    }
}

#启动镜像
docker run -d --name logstash  -p 5044:5044 --log-driver json-file --log-opt max-size=100m --log-opt max-file=2 -v /data/elk/logstash/logstash.yml:/usr/share/logstash/config/logstash.yml -v /data/elk/logstash/conf.d/:/data/docker/logstash/conf.d/ logstash:7.6.2

整合kafka后整体运行流程

springcloud服务集成ELK日志系统（建议放入公共模块中操作，业务模块直接导入即可，请勿导入其他日志框架避免出现冲突！）

1.导入maven依赖

<!--以下依赖是logback接入kafka所需依赖-->
<dependency>
   <groupId>com.github.danielwegener</groupId>
   <artifactId>logback-kafka-appender</artifactId>
   <version>0.2.0-RC2</version>
</dependency>
<dependency>
   <groupId>ch.qos.logback</groupId>
   <artifactId>logback-classic</artifactId>
</dependency>
<dependency>
   <groupId>ch.qos.logback</groupId>
   <artifactId>logback-core</artifactId>
</dependency>
<!-- kafka依赖 -->
<dependency>
    <groupId>org.springframework.kafka</groupId>
    <artifactId>spring-kafka</artifactId>
</dependency>
<dependency>
    <groupId>net.logstash.logback</groupId>
    <artifactId>logstash-logback-encoder</artifactId>
    <version>6.6</version>
</dependency>

2.创建LogIpConfig.java,后面打印日志时实现ip地址打印

package com.zxj.elk;

import ch.qos.logback.classic.pattern.ClassicConverter;
import ch.qos.logback.classic.spi.ILoggingEvent;
import lombok.extern.slf4j.Slf4j;

import java.net.InetAddress;
import java.net.UnknownHostException;

/**
 * 获取ip地址
 *
 * @author zhangxuejie
 * @email 898953030@qq.com
 * @date 2022-7-10 16:19
 */
@Slf4j
public class LogIpConfig extends ClassicConverter {
    private static String webIP;

    static {
        try {
            webIP = InetAddress.getLocalHost().getHostAddress();
        } catch (UnknownHostException e) {
            log.error("获取日志Ip异常", e);
            webIP = null;
        }
    }

    @Override
    public String convert(ILoggingEvent event) {
        return webIP;
    }
}

3.创建logback-spring.xml文件（将日志发布到kafka）

<?xml version="1.0" encoding="UTF-8"?>
<!-- scan：当此属性设置为true时，配置文件如果发生改变，将会被重新加载，默认值为true。 scanPeriod：设置监测配置文件是否有修改的时间间隔，如果没有给出时间单位，
	默认单位是毫秒当scan为true时，此属性生效。默认的时间间隔为1分钟。 debug：当此属性设置为true时，将打印出logback内部日志信息，实时查看logback运行状态。
	默认值为false。 -->
<!-- <configuration scan="false" scanPeriod="60 seconds" debug="false"> -->
<configuration debug="false">
    <!-- 实现日志ip打印 -->
    <conversionRule conversionWord="ip" converterClass="com.zxj.elk.LogIpConfig"/>
    <!--从配置文件获取app名称-->
    <springProperty scope="context" name="springAppName" source="spring.application.name"/>

    <!-- 彩色日志(IDE下载插件才可以生效) -->
    <!-- 彩色日志依赖的渲染类 -->
    <conversionRule conversionWord="clr"
                    converterClass="org.springframework.boot.logging.logback.ColorConverter"/>
    <conversionRule conversionWord="wex"
                    converterClass="org.springframework.boot.logging.logback.WhitespaceThrowableProxyConverter"/>
    <conversionRule conversionWord="wEx"
                    converterClass="org.springframework.boot.logging.logback.ExtendedWhitespaceThrowableProxyConverter"/>
    <!-- 彩色日志格式 -->
    <property name="CONSOLE_LOG_PATTERN"
              value="${CONSOLE_LOG_PATTERN:-%clr(%d{yyyy-MM-dd HH:mm:ss.SSS}){faint} %clr(${LOG_LEVEL_PATTERN:-%5p}) %clr(${PID:- }){magenta} %clr(---){faint} %clr([%15.15t]){faint} %clr(%-40.40logger{39}){cyan} %clr(:){faint} %m%n${LOG_EXCEPTION_CONVERSION_WORD:-%wEx}}"/>

    <!-- ConsoleAppender 表示控制台输出 -->
    <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
        <!-- 日志输出格式： %d表示日期时间， %thread表示线程名， %-5level：级别从左显示5个字符宽度, %logger{50}
            表示logger名字最长50个字符，否则按照句点分割。 %msg：日志消息， %n是换行符 -->
        <encoder>
            <pattern>${CONSOLE_LOG_PATTERN}
            </pattern>
        </encoder>
    </appender>

    <appender name="kafkaAppender" class="com.github.danielwegener.logback.kafka.KafkaAppender">
        <encoder charset="UTF-8" class="net.logstash.logback.encoder.LoggingEventCompositeJsonEncoder">
            <providers>
                <timestamp>
                    <timeZone>GMT+8</timeZone>
                </timestamp>
                <!--必须用两个pattern，不然日志不完整-->
                <pattern>
                    <pattern>
                        {
                        "ip":"%ip",
                        "severity": "%level",
                        "service": "${springAppName:-}",
                        "trace": "%X{X-B3-TraceId:-}",
                        "span": "%X{X-B3-SpanId:-}",
                        "parent": "%X{X-B3-ParentSpanId:-}",
                        "exportable": "%X{X-Span-Export:-}",
                        "pid": "${PID:-}",
                        "thread": "%thread",
                        "class": "%logger{40}",
                        "rest": "%message",
                        "stack_trace": "%exception{30}"
                        }
                    </pattern>
                </pattern>
            </providers>
        </encoder>
        <!-- 此处即为kafka的主题Topic名称-->
        <topic>${springAppName:-}-log</topic>
        <!-- we don't care how the log messages will be partitioned  -->
        <keyingStrategy class="com.github.danielwegener.logback.kafka.keying.NoKeyKeyingStrategy"/>
        <!-- use async delivery. the application threads are not blocked by logging -->
        <deliveryStrategy class="com.github.danielwegener.logback.kafka.delivery.AsynchronousDeliveryStrategy"/>
        <!-- each <producerConfig> translates to regular kafka-client config (format: key=value) -->
        <!-- producer configs are documented here: https://kafka.apache.org/documentation.html#newproducerconfigs -->
        <!-- bootstrap.servers is the only mandatory producerConfig -->
        <producerConfig>bootstrap.servers=localhost:9092</producerConfig>
        <!-- don't wait for a broker to ack the reception of a batch.  -->
        <producerConfig>acks=0</producerConfig>
        <!-- wait up to 1000ms and collect log messages before sending them as a batch -->
        <producerConfig>linger.ms=1000</producerConfig>
        <!-- even if the producer buffer runs full, do not block the application but start to drop messages -->
        <producerConfig>max.block.ms=0</producerConfig>
        <!-- define a client-id that you use to identify yourself against the kafka broker -->
        <producerConfig>client.id=0</producerConfig>
        <!-- this is the fallback appender if kafka is not available. -->
        <appender-ref ref="CONSOLE"/>
    </appender>

    <root level="info">
        <appender-ref ref="CONSOLE"/>
        <appender-ref ref="kafkaAppender"/>
    </root>
</configuration>

访问elasticsearch-head(ip:9100)查看索引是否创建成功,如果有就说明成功了。