十分钟搭建elk日志分析系统并使用

部署elk：

如果你已经有Elasticsearch、Kibana、Logstash可以卸载掉

配置elasticsearch准备：

-------------------------------------------------------------------------------------------------------------------------------

#新建目录

mkdir /usr/local/docker/elk

增加es目录

mkdir -p /usr/local/docker/elk/elasticsearch/{logs,data,config,plugins}

进入文件夹

cd /usr/local/docker/elk/elasticsearch

赋权目录data

chmod -R 777 data

进入目录config

cd config

编写文件elasticsearch

elasticsearch.yml内容

cluster.name: "docker-cluster"
network.host: 0.0.0.0
http.port: 9200
# 开启es跨域
http.cors.enabled: true
http.cors.allow-origin: "*"
http.cors.allow-headers: Authorization
# 开启安全控制
xpack.security.enabled: true
xpack.security.transport.ssl.enabled: true

配置kibana准备：

创建文件夹kibana

mkdir -p /usr/local/docker/elk/kibana/config

进入文件夹kibana

cd /usr/local/docker/elk/kibana

进入config目录

cd config

编写文件kibana.yml

vim kibana.yml

kibana.yml内容：

server.name: kibana
server.host: "0.0.0.0"
server.publicBaseUrl: "http://kibana:5601" #不用更改配置文件配置好了
elasticsearch.hosts: [ "http://elasticsearch:9200" ] ##不用更改配置文件配置好了
xpack.monitoring.ui.container.elasticsearch.enabled: true
elasticsearch.username: "elastic" #用户账号
elasticsearch.password: "123456"  #用户密码
i18n.locale: zh-CN

配置logstash准备:

创建文件夹logstash

mkdir -p /usr/local/docker/elk/logstash/{data,config}

创建目录small-tools

mkdir -p /usr/local/docker/elk/logstash/config/small-tools

进入对应目录

cd /usr/local/docker/elk/logstash

赋权

chmod 777 data

进入目录

cd /usr/local/docker/elk/logstash/config

编写配置文件

vim logstash.yml

logstash.yml内容:

http.host: "0.0.0.0"
xpack.monitoring.enabled: true
xpack.monitoring.elasticsearch.hosts: [ "http://elasticsearch:9200" ] 
xpack.monitoring.elasticsearch.username: "elastic"  #用户账号
xpack.monitoring.elasticsearch.password: "123456"   #用户密码

进入small-tools下增加demo项目监控配置文件

cd /usr/local/docker/elk/logstash/config/small-tools

编写 demo.config

vim demo.config

demo.config内容:

input { #输入

    tcp {
        mode => "server"
        host => "0.0.0.0"   # 允许任意主机发送日志
        type => "cjq"      # 设定type以区分每个输入源
        port => 9999
        codec => json_lines # 数据格式
    }

}

filter {
    mutate {
        # 导入之过滤字段
        remove_field => ["LOG_MAX_HISTORY_DAY", "LOG_HOME", "APP_NAME"]
        remove_field => ["@version", "_score", "port", "level_value", "tags", "_type", "host"]
    }
}

output { #输出-控制台
    stdout{
        codec => rubydebug
    }
}

output { #输出-es

    if [type] == "cjq" {
        elasticsearch {
            action => "index"                       # 输出时创建映射
            hosts  => "http://elasticsearch:9200"   # ES地址和端口
            user => "elastic"                       # ES用户名
            password => "123456"                    # ES密码
            index  => "cjq-%{+YYYY.MM.dd}"         # 指定索引名-按天
            codec  => "json"
        }
    }

}

这个index内容可以更改

elk目录下增加docker-compose文件

cd /usr/local/docker/elk

编写 docker-compose.yml

vim docker-compose.yml

docker-compose.yml内容：

如果想更换版本image可以自行查找进行更改

version: '3.0'
networks:
  elk:
    driver: bridge
services:
  elasticsearch:
    image: registry.cn-hangzhou.aliyuncs.com/zhengqing/elasticsearch:7.14.1
    container_name: elk_elasticsearch
    restart: unless-stopped
    volumes:
      - "/usr/local/docker/elk/elasticsearch/data:/usr/share/elasticsearch/data"
      - "/usr/local/docker/elk/elasticsearch/logs:/usr/share/elasticsearch/logs"
      - "/usr/local/docker/elk/elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml"
      - "/usr/local/docker/elk/elasticsearch/plugins:/usr/share/elasticsearch/plugins"
    environment:
      TZ: Asia/Shanghai
      LANG: en_US.UTF-8
      TAKE_FILE_OWNERSHIP: "true"  # 权限
      discovery.type: single-node
      ES_JAVA_OPTS: "-Xmx512m -Xms512m"
      ELASTIC_PASSWORD: "123456" # elastic账号密码
    ports:
      - "9200:9200"
      - "9300:9300"
    networks:
      - elk

  kibana:
    image: registry.cn-hangzhou.aliyuncs.com/zhengqing/kibana:7.14.1
    container_name: elk_kibana
    restart: unless-stopped
    volumes:
      - "/usr/local/docker/elk/kibana/config/kibana.yml:/usr/share/kibana/config/kibana.yml"
    ports:
      - "5601:5601"
    depends_on:
      - elasticsearch
    links:
      - elasticsearch
    networks:
      - elk

  logstash:
    image: registry.cn-hangzhou.aliyuncs.com/zhengqing/logstash:7.14.1
    container_name: elk_logstash
    restart: unless-stopped
    environment:
      LS_JAVA_OPTS: "-Xmx512m -Xms512m"
    volumes:
      - "/usr/local/docker/elk/logstash/data:/usr/share/logstash/data"
      - "/usr/local/docker/elk/logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml"
      - "/usr/local/docker/elk/logstash/config/small-tools:/usr/share/logstash/config/small-tools"
    command: logstash -f /usr/share/logstash/config/small-tools
    ports:
      - "9600:9600"
      - "9999:9999"
    depends_on:
      - elasticsearch
    networks:
      - elk

编排elk：

docker compose up -d

SpringBoot整合logstash：

依赖：

<dependency>
    <groupId>net.logstash.logback</groupId>
    <artifactId>logstash-logback-encoder</artifactId>
    <version>6.6</version>
</dependency>

在resources目录下添加logback.xml:

<?xml version="1.0" encoding="UTF-8"?>
<configuration>
    <appender name="LOGSTASH" class="net.logstash.logback.appender.LogstashTcpSocketAppender">
        <!-- logstash日志收集服务地址 -->
        <destination>自己ip地址:9999</destination>
        <encoder charset="UTF-8" class="net.logstash.logback.encoder.LogstashEncoder">
            <!-- 日志自定义字段，随日志内容一起进行存储，方便在日志查看端kibana中按此字段进行检索 -->
<!--            <customFields>{"Cjq":"dev-iot-service"}</customFields>-->
        </encoder>
    </appender>

    <property name="CONSOLE_PATTERN"
              value="===%date{yyyy-MM-dd HH:mm:ss} PID=%relative %highlight(%-5level) [%thread] %logger Line:%-3L - %msg%n"/>
    <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
        <!--encoder 默认配置为PatternLayoutEncoder-->
        <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
            <pattern>${CONSOLE_PATTERN}</pattern>
            <charset>utf-8</charset>
        </encoder>
        <!--此日志appender是为开发使用，只配置最底级别，控制台输出的日志级别是大于或等于此级别的日志信息-->
        <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
            <level>debug</level>
        </filter>
    </appender>

    <root level="INFO">
        <appender-ref ref="LOGSTASH"/>
        <appender-ref ref="CONSOLE"/>
    </root>
</configuration>

上述代码中需要注意的：

Cjq可以根据自己名称、项目名称或者模块名称进行定义

 启动项目来查看日志步骤：

初始进入会进行登录，账号密码默认是elastic、123456

之后进入5601如果需要登入密码如上

点击索引模式

创建索引

可以直接使用上一步的那个名称然后进行下一步

 时间选择

 查看日志

 查看多个项目

字段后面有加号可以进行点击