Logstash指定部分output的document_id

需求

有两个Filebeat进行日志采集，通过同一个logstash输出至ElasticSearch，其中一个采集器a中需要指定一个字段key_id为document_id，以确保以此字段进行重复过滤，另一个采集器b使用默认生成的document_id。

logstash配置

input {
    beats {
        port => 5044
        host => "0.0.0.0"
        ssl => false
    }
}

filter{
    if [fields][app_name] == "a"{
        grok {
            patterns_dir => ["/opt/logstash/patterns"]
            match => {
                "message" => "..."
            }
            overwrite => [ "message" ]
        }
        date {
            match => [
                "[time][log]", "ISO8601"
            ]
        }
        prune {
            whitelist_names => [ "@timestamp", "key_id"...]//保留字段中含有key_id
        }
    }
    if [fields][app_name] == "b"{
        grok {
            patterns_dir => ["/opt/logstash/patterns"]
            match => {
                "message" => "(?<message>.*)"
            }
            overwrite => [ "message" ]
        }
        date {
            match => [
                "[time][log]", "ISO8601"
            ]
        }
        prune {
            whitelist_names => [ "@timestamp", "message"]//保留字段中无key_id
        }
    }
}

output{
    if [key_id] {//若含有key_id的字段，对此字段进行转换为document_id，以确保对此字段的唯一性进行重复消息覆盖
        elasticsearch{
            hosts=>["es01:9200"]
            manage_template => false
            index => "%{[fields][app_name]}"
            document_type => "%{[@metadata][type]}"
            document_id => "%{key_id}"
        }
    } else {//一般不含key_id的输出不对document_id进行转换
        elasticsearch{
            hosts=>["es01:9200"]
            manage_template => false
            index => "%{[fields][app_name]}"
            document_type => "%{[@metadata][type]}"
        }
    }
}