elk一、docker部署filebeat抓取容器日志配置 (output 到redis)

官方配置参考
docker参考地址

创建文件夹以及文件

drwxr-xr-x. 2 root root  54 7月  15 13:24 config
-rwxr-xr-x. 1 root root 412 6月  28 17:15 docker-compose.yml
[root@mdm-dev2 filebeat]# pwd
/home/filebeat
[root@mdm-dev2 filebeat]#

docker-compose.yml文件内容如下（logging是给日志中添加标签）

version: '2'

services:
  filebeat:
    image: prima/filebeat:5.6.1
    hostname: STdev2
    labels:
      service: STdev2-filebeat
    logging:
      options:
        labels: "service"
    restart: always
    volumes:
      - /home/filebeat/config/filebeat.yml:/filebeat.yml
      - ~/dockerdata/filebeat:/data
      - /var/lib/docker/containers:/var/lib/docker/containers
      - /etc/localtime:/etc/localtime

-rw-r--r--. 1 root root 2398 6月  23 13:51 filebeat.template.json
-rwx--x--x. 1 root root 6072 7月   9 11:46 filebeat.yml
[root@mdm-dev2 config]# pwd
/home/filebeat/config

filebeat.yml文件配置如下

###################### Filebeat Configuration Example #########################

# This file is an example configuration file highlighting only the most common
# options. The filebeat.full.yml file from the same directory contains all the
# supported options with more comments. You can use it as a reference.
#
# You can find the full configuration reference here:
# https://www.elastic.co/guide/en/beats/filebeat/index.html

#=========================== Filebeat prospectors =============================

filebeat.prospectors:

# Each - is a prospector. Most options can be set at the prospector level, so
# you can use different prospectors for various configurations.
# Below are the prospector specific configurations.

- input_type: log
  # Paths that should be crawled and fetched. Glob based paths.
  paths:
#    - /var/log/messages
#    - /var/log/*.log
#    - /var/log/containers/*.log
    - /var/lib/docker/containers/*/*.log
    #- c:\programdata\elasticsearch\logs\*
  tags: ["st-dev2"]

  document_type: list
  # Exclude lines. A list of regular expressions to match. It drops the lines that are
  # matching any regular expression from the list.
  #exclude_lines: ["^DBG"]

  # Include lines. A list of regular expressions to match. It exports the lines that are
  # matching any regular expression from the list.
  #include_lines: ["^ERR", "^WARN"]

  # Exclude files. A list of regular expressions to match. Filebeat drops the files that
  # are matching any regular expression from the list. By default, no files are dropped.
 # exclude_files: [".gz$"]

  # Optional additional fields. These field can be freely picked
  # to add additional information to the crawled log files for filtering
  #fields:
  #  level: debug
  #  review: 1

  #fields:
  #  level: INFO

  ### Multiline options

  # Mutiline can be used for log messages spanning multiple lines. This is common
  # for Java Stack Traces or C-Line Continuation

  # The regexp Pattern that has to be matched. The example pattern matches all lines starting with [
  #multiline.pattern: ^\[
  multiline.pattern: '^Mon|Tue|Web|Thu|Fri|Sat|Sun [[:space:]] Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec'
  # Defines if the pattern set under pattern should be negated or not. Default is false.
 # multiline.negate: true

  # Match can be set to "after" or "before". It is used to define if lines should be append to a pattern
  # that was (not) matched before or after or as long as a pattern is not matched based on negate.
  # Note: After is the equivalent to previous and before is the equivalent to to next in Logstash
#  multiline.match: after

  json.keys_under_root: true
  # 因为docker使用的log driver是json-file，因此采集到的日志格式是json格式，设置为true之后，filebeat会将日志进行json_decode处理
  json.add_error_key: true
  #如果启用此设置，则在出现JSON解组错误或配置中定义了message_key但无法使用的情况下，Filebeat将添加“error.message”和“error.type：json”键。
  json.message_key: log
  #一个可选的配置设置，用于指定应用行筛选和多行设置的JSON密钥。 如果指定，键必须位于JSON对象的顶层，且与键关联的值必须是字符串，否则不会发生过滤或多行聚合。
  tail_files: true
  # 将error日志合并到一行
 # multiline.pattern: '^([0-9]{4}|[0-9]{2})-[0-9]{2}'
 # multiline.pattern: ^(\s|[A-Z][a-z]|-)
  multiline.negate: true
  multiline.match: after
  multiline.timeout: 10s
  #可以合并为一个事件的最大行数。如果多行消息包含多个max_lines，则所有其他行都将被丢弃。默认值为500。
  multiline.max_lines: 10000
#================================ General =====================================

# The name of the shipper that publishes the network data. It can be used to group
# all the transactions sent by a single shipper in the web interface.
#name:

# The tags of the shipper are included in their own field with each
# transaction published.
#tags: ["service-X", "web-tier"]

# Optional fields that you can specify to add additional information to the
# output.
#fields:
#  env: staging

#================================ Outputs =====================================

# Configure what outputs to use when sending the data collected by the beat.
# Multiple outputs may be used.

#-------------------------- Elasticsearch output ------------------------------
#output.elasticsearch:
  # Array of hosts to connect to.
#  hosts: ["localhost:9200"]

  # Optional protocol and basic auth credentials.
  #protocol: "https"
  #username: "elastic"
  #password: "changeme"

output.redis:
  # Array of hosts to connect to.
   hosts: ["192.168.180.6"]
   port: 6379
   password: "sinoeyes"
   key: "sinoeyes-io"
   db: 4
   timeout: 5
   dataytpe: "list"

   template.name: "filebeat"
   template.path: "filebeat.template.json"
   template.overwrite: false
#----------------------------- Logstash output --------------------------------
#output.logstash:
  # The Logstash hosts
  #hosts: ["localhost:5044"]

  # Optional SSL. By default is off.
  # List of root certificates for HTTPS server verifications
  #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]

  # Certificate for SSL client authentication
  #ssl.certificate: "/etc/pki/client/cert.pem"

  # Client Certificate Key
  #ssl.key: "/etc/pki/client/cert.key"

#================================ Logging =====================================

# Sets log level. The default log level is info.
# Available log levels are: critical, error, warning, info, debug
#logging.level: debug
#logging.level: critical, error, warning, info, debug
#logging.level: info
# At debug level, you can selectively enable logging only for some components.
# To enable all selectors use ["*"]. Examples of other selectors are "beat",
# "publish", "service".
logging.selectors: ["*"]

#==================== Elasticsearch template setting ==========================

#setup.template.name: "filebeat.template.json"
#setup.template.fields: "filebeat.template.json"
#setup.template.overwrite: true
#setup.template.enabled: false

# 过滤掉一些不必要字段#
processors:
- drop_fields:
    fields: ["input_type", "offset", "stream", "beat"]

filebeat.template.json文件内容如下

{
  "mappings": {
    "_default_": {
      "_all": {
        "norms": false
      },
      "_meta": {
        "version": "5.1.2"
      },
      "dynamic_templates": [
        {
          "strings_as_keyword": {
            "mapping": {
              "ignore_above": 1024,
              "type": "keyword"
            },
            "match_mapping_type": "string"
          }
        }
      ],
      "properties": {
        "@timestamp": {
          "type": "date"
        },
        "beat": {
          "properties": {
            "hostname": {
              "ignore_above": 1024,
              "type": "keyword"
            },
            "name": {
              "ignore_above": 1024,
              "type": "keyword"
            },
            "version": {
              "ignore_above": 1024,
              "type": "keyword"
            }
          }
        },
        "input_type": {
          "ignore_above": 1024,
          "type": "keyword"
        },
        "message": {
          "norms": false,
          "type": "text"
        },
        "meta": {
          "properties": {
            "cloud": {
              "properties": {
                "availability_zone": {
                  "ignore_above": 1024,
                  "type": "keyword"
                },
                "instance_id": {
                  "ignore_above": 1024,
                  "type": "keyword"
                },
                "machine_type": {
                  "ignore_above": 1024,
                  "type": "keyword"
                },
                "project_id": {
                  "ignore_above": 1024,
                  "type": "keyword"
                },
                "provider": {
                  "ignore_above": 1024,
                  "type": "keyword"
                },
                "region": {
                  "ignore_above": 1024,
                  "type": "keyword"
                }
              }
            }
          }
        },
        "offset": {
          "type": "long"
        },
        "source": {
          "ignore_above": 1024,
          "type": "keyword"
        },
        "tags": {
          "ignore_above": 1024,
          "type": "keyword"
        },
        "type": {
          "ignore_above": 1024,
          "type": "keyword"
        }
      }
    }
  },
  "order": 0,
  "settings": {
    "index.refresh_interval": "5s"
  },
  "template": "filebeat-*"
}

方法二、

官方镜像地址

可能出现的错误：

ERR Connecting error publishing events (retrying): dial tcp 192.168.180.6:6379: i/o timeout

可能产生的原因之一

CONTAINER ID        IMAGE                                          COMMAND                  CREATED             STATUS                      PORTS               NAMES
b6053581ee50        prima/filebeat:5.6.1                           "/docker-entrypoint.…"   8 minutes ago       Exited (0) 7 minutes ago                        filebeat_filebeat_1