graylog使用Sidecars方式收集springboot程序的日志

1、部署graylog后台服务

使用docker-compose启动三个服务程序，包括graylog、mongodb、opensearch。

docker-compose.yml内容如下

version: '3'
services:
# MongoDB: https://hub.docker.com/_/mongo/
  mongodb:
    image: mongo:6.0.14
    privileged: true
    networks:
      - graylog

  opensearch:
    image: "opensearchproject/opensearch:2.12.0"
    environment:
      - "OPENSEARCH_JAVA_OPTS=-Xms1g -Xmx1g"
      - "bootstrap.memory_lock=true"
      - "discovery.type=single-node"
      - "action.auto_create_index=false"
      - "plugins.security.ssl.http.enabled=false"
      - "plugins.security.disabled=true"
      # Can generate a password for `OPENSEARCH_INITIAL_ADMIN_PASSWORD` using a linux device via:
      # tr -dc A-Z-a-z-0-9_@#%^-_=+ < /dev/urandom  | head -c${1:-32}
      - OPENSEARCH_INITIAL_ADMIN_PASSWORD=+_8r#wliY3Pv5-HMIf4qzXImYzZf-M=M
    privileged: true
    ulimits:
      memlock:
        hard: -1
        soft: -1
      nofile:
        soft: 65536
        hard: 65536
    restart: "on-failure"
    networks:
        - graylog

  # Graylog: https://hub.docker.com/r/graylog/graylog/
  graylog:
    image: graylog/graylog:5.2
    environment:
    - GRAYLOG_NODE_ID_FILE=/usr/share/graylog/data/config/node-id
    - GRAYLOG_HTTP_BIND_ADDRESS=0.0.0.0:9000
    - GRAYLOG_ELASTICSEARCH_HOSTS=http://opensearch:9200
    - GRAYLOG_MONGODB_URI=mongodb://mongodb:27017/graylog
    # CHANGE ME (must be at least 16 characters)!
    - GRAYLOG_PASSWORD_SECRET=somepasswordpepper
    # Password: admin
    - GRAYLOG_ROOT_PASSWORD_SHA2=8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918
    - GRAYLOG_HTTP_EXTERNAL_URI=http://127.0.0.1:9000/
    privileged: true
    entrypoint: /usr/bin/tini -- wait-for-it elasticsearch:9200 --  /docker-entrypoint.sh
    networks:
      - graylog
    restart: always
    depends_on:
      - mongodb
      - opensearch
    ports:
      # Graylog web interface and REST API
      - 9000:9000
      # Syslog TCP
      - 1514:1514
      # Syslog UDP
      - 1514:1514/udp
      # GELF TCP
      - 12201:12201
      # GELF UDP
      - 12201:12201/udp
      - 5044:5044

networks:
  graylog:
    driver: bridge

2、部署k8s上graylog的后台服务 

 graylog2-service.yaml

启动运行kubectl apply -f graylog2-service.yaml

apiVersion: v1
kind: Service
metadata:
  labels:
    app: graylog2
  name: graylog2
  namespace: example
spec:
  type: ClusterIP
  ports:
    - name: rest
      port: 9000
      protocol: TCP
    - name: tsyslog
      port: 1514
      protocol: TCP
    - name: usyslog
      port: 1514
      protocol: UDP
    - name: tgelf
      port: 12201
      protocol: TCP
    - name: ugelf
      port: 12201
      protocol: UDP
    - name: tbeat
      port: 5044
      protocol: TCP
    - name: ubeat
      port: 5044
      protocol: UDP
---
apiVersion: v1
kind: Endpoints
metadata:
  labels:
    app: graylog2
  name: graylog2
  namespace: example
subsets:
  - addresses:
      - ip: 上面执行docker-compose up -d的主机ip
    ports:
      - name: rest
        port: 9000
        protocol: TCP
      - name: tsyslog
        port: 1514
        protocol: TCP
      - name: usyslog
        port: 1514
        protocol: UDP
      - name: tgelf
        port: 12201
        protocol: TCP
      - name: ugelf
        port: 12201
        protocol: UDP
      - name: tbeat
        port: 5044
        protocol: TCP
      - name: ubeat
        port: 5044
        protocol: UDP

3、登录graylog后台配置Sidecars信息

登录地址

http://ip:9000

登录用户名/密码

admin/admin

顶部菜单选择【System】中的之后一个Sidecars，点击【Create or reuse a token for the graylog-sidecar user】

其中Token Name自定义填写，点击Create Token按钮，会生成token信息。

生成的Token需要复制保留，下面file-service.yaml文件需要使用到

查看节点id信息，在System元素处，会看见节点id信息：Node ID

节点ID需要复制保留，下面file-service.yaml文件需要使用到

4、部署springboot程序在k8s上

file-service.yaml，启动运行kubectl apply -f file-service.yaml

apiVersion: v1
kind: Service
metadata:
  name: file-service
  namespace: example
  labels:
    service: file-service
spec:
  ports:
    - name: http
      port: 8080
      protocol: TCP
      targetPort: 8080
      nodePort: 30250
  selector:
    app: file-service
  type: NodePort
---
apiVersion: v1
kind: Service
metadata:
  name: file-service-headless
  namespace: example
  labels:
    app: file-service
spec:
  ports:
    - name: http
      port: 8080
      targetPort: 8080
  clusterIP: None
  selector:
    app: file-service
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: file-service
  namespace: example
spec:
  replicas: 1
  serviceName: file-service-headless
  selector:
    matchLabels:
      app: file-service
  template:
    metadata:
      labels:
        app: file-service
        release: default
    spec:
      restartPolicy: Always
      containers:
        - name: file-service
          resources:
            requests:
              ephemeral-storage: 2048Mi
            limits:
              ephemeral-storage: 2048Mi
          imagePullPolicy: IfNotPresent
          image: file-service:v1          
          ports:
            - name: http
              containerPort: 8080
          volumeMounts:
            - name: volume-localtime
              mountPath: /etc/localtime              
            - name: graylog2-logs
              #springboot程序容器内产生日志的目录
              mountPath: /opt/file-service/logs   
        - name: sidecar-collector-logs
          imagePullPolicy: IfNotPresent
          image: graylog-log-sidecar-collector:latest          
          env:
            - name: GS_SERVER_URL
              #定义graylog后台服务的地址
              value: "http://graylog2:9000/api/"
              #定义graylog服务的节点id，取值来自上一步复制的内容
            - name: GS_NODE_ID
              value: "5861f0cb-e128-4f2e-a17b-3e42f8bff6af"
              #节点名称自定义
            - name: GS_NODE_NAME
              value: "sidecar-collector-logs-file-service"
              #使用的token，取值来自上一步复制的内容
            - name: GS_SERVER_API_TOKEN
              value: "17b7haug3bvflmtuj23e34eg9raen6bsmcppdo1aluls7s05juvn"
              #采集器容器的目录,通过挂载的方式GS_LIST_LOG_FILES与运行springboot程序的graylog2-logs目录进行关联
            - name: GS_LIST_LOG_FILES
              value: "/graylog2-logs"
          volumeMounts:
            - name: graylog2-logs
              mountPath: /graylog2-logs
      volumes:
        - name: volume-localtime
          hostPath:
            path: /etc/localtime
            type: ''
        #同时映射了宿主机目录，如果产生的日志不够，可以在这个文件夹内手动添加*.log日志。
        - name: graylog2-logs
          hostPath:
            path: /home/volume
            type: DirectoryOrCreate 

5、配置Sidecars 

springboot正常启动后，会自动注册到graylog上。

下面是两个容器，一个复制运行程序，另一个负责日志收集上报。

sidecars的运行状态已经是Running，但此时还需要配置file beat信息

点击名称链接，进入新页面，可以看到加载的日志目录信息

指派filebeat配置

编辑Configuration下的Log Collectors，选项下图的内容进行编辑，Executable Path要改成实际的目录，

graylog-log-sidecar-collector:latest  镜像我将filebeat放到了

/usr/share/filebeat/bin/filebeat，需要根据实际情况进行修改。点击Update Collectors进行保存。

 Configurations中选择下面内容进行编辑

模板配置内容

# Needed for Graylog
fields_under_root: true
fields.collector_node_id: ${sidecar.nodeName}
fields.gl2_source_collector: ${sidecar.nodeId}

output.logstash:
   hosts: ["${user.graylog_host}:5044"]

   #hosts: ["${user.graylog_host}:12201"]
path:
   data: ${sidecar.spoolDir!"/var/lib/graylog-sidecar/collectors/filebeat"}/data
   logs: ${sidecar.spoolDir!"/var/lib/graylog-sidecar/collectors/filebeat"}/log

filebeat.inputs:
- type: log
  id: sidecar-collector-logs-file-service
  enabled: true
  paths:
  - /graylog2-logs/*.log
  fields_under_root: true
  fields:
      event_source_product: springboot

修改右侧的环境变量信息 ，k8s中使用域名访问服务。

 重启Sidecar插件服务

观察容器内日志

6、配置Input 

填写名称即可，选择Global全局的。

7、查看收集日志效果 

8、Sidecar Dockerfile

FROM debian:buster-slim

LABEL maintainer 'Markus Gulden <mg@gulden.consulting>'

RUN apt-get update && apt-get install -y openssl libapr1 libdbi1 libexpat1 ca-certificates

ENV SIDECAR_BINARY_URL https://github.com/Graylog2/collector-sidecar/releases/download/1.5.0/graylog-sidecar_1.5.0-1_amd64.deb
ENV FILEBEAT_BINARY_URL https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-7.1.1-amd64.deb

RUN apt-get install -y --no-install-recommends curl && curl -Lo sidecar.deb ${SIDECAR_BINARY_URL} && dpkg -i sidecar.deb && rm sidecar.deb && curl -Lo filebeat.deb ${FILEBEAT_BINARY_URL} && dpkg -i filebeat.deb && rm filebeat.deb && apt-get purge -y --auto-remove curl

#GS_LIST_LOG_FILES="[]"

ENV GS_UPDATE_INTERVAL=10 \
    GS_TLS_SKIP_VERIFY="false" \
    GS_SEND_STATUS="true" \    
    GS_CACHE_PATH="/var/cache/graylog-sidecar" \
    GS_COLLECTOR_CONFIGURATION_DIRECTORY="/var/lib/graylog-sidecar/generated" \
    GS_LOG_PATH="/var/log/graylog-sidecar" \
    GS_LOG_ROTATE_MAX_FILE_SIZE="1MiB" \
    GS_LOG_ROTATE_KEEP_FILES=100 \
    GS_COLLECTOR_BINARIES_WHITELIST="["/usr/bin/filebeat", "/usr/bin/packetbeat", "/usr/bin/metricbeat", "/usr/bin/heartbeat", "/usr/bin/auditbeat", "/usr/bin/journalbeat", "/usr/share/filebeat/bin/filebeat", "/usr/share/packetbeat/bin/packetbeat", "/usr/share/metricbeat/bin/metricbeat", "/usr/share/heartbeat/bin/heartbeat", "/usr/share/auditbeat/bin/auditbeat", "/usr/share/journalbeat/bin/journalbeat", "/usr/bin/nxlog", "/opt/nxlog/bin/nxlog"]"
ADD ./data /data
CMD /usr/bin/graylog-sidecar -c /data/sidecar.yml