elk配置安装

1.filebeat配置

1.1下载

curl -L -O https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-7.5.1-linux-x86_64.tar.gz

1.2配置

修改filebeat.yml配置文件

filebeat.inputs:
- type: log
  paths:
    - /log*
  tags: ["tag"]
output.logstash:
  hosts: [""]

或者直接打入到es:

filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /home/*
  #解析json到es，否则会变成json字符串存入message
  json.keys_under_root: true
  json.overwrite_keys: true

setup.template.settings:
  index.number_of_shards: 3
  index.number_of_replicas: 0
  index.codec: best_compression

output.elasticsearch:
  hosts: ["host","host"]
  username: "elastic"
  password: "pass"
  indices:
   - index: "index-%{+yyyyMM}"

processors:
  - drop_fields:
      fields: ["log","input","ecs","host","agent","cloud"]

主要是input和output

1.3启动

在filebeat目录下执行
nohup ./filebeat -e -c filebeat.yml &

2.logstash安装

2.1下载

wget https://artifacts.elastic.co/downloads/logstash/logstash-7.5.1.tar.gz

2.2配置

在config目录下创建conf.d目录，在conf.d目录下创建文件logstash.yml

input {
        beats {
                port => "5044"
        }
}

filter {
    if "aa" in [tags] {
        json {
            source => "message"
            remove_field => ["message", "beat", "offset", "[resp][data]"]
        }
        date {
            match => ["req_timestamp", "UNIX_MS"]
            remove_field => ["req_timestamp"]
        }
    } else if "bb" in [tags] {
        json {
            source => "message"
            remove_field => ["message", "beat", "offset", "[resp][data]","agent","ecs","log","input","@version"]
        }
        date {
            match => ["req_timestamp", "UNIX_MS"]
            remove_field => ["req_timestamp"]
        }
         ruby {
            code => "event.set('index_month', event.get('@timestamp').time.localtime.strftime('%Y%m'))"
       }
    } else if "cc" in[tags] {
        json {
            source => "message"
            remove_field => ["message", "beat", "offset"]
        }
        date {
            match => ["req_timestamp", "UNIX_MS"]
            remove_field => ["req_timestamp"]
        }
    } else if "dd" in [tags]{
         json {
            source => "message"
            remove_field => ["message", "beat", "offset","agent","input","ecs","@version","log"]
        }
     }else if "ee" in [tags]{
        json {
            source => "message"
            remove_field => ["message", "beat", "offset","agent","input","ecs","@version","log"]
        }
        date {
            match => ["req_timestamp", "UNIX_MS"]
            remove_field => ["req_timestamp"]
        }
         ruby {
            code => "event.set('index_month', event.get('@timestamp').time.localtime.strftime('%Y%m'))"
       }
    }else if "dd" in [tags]{

         json {
            source => "message"
            remove_field => ["message", "beat", "offset","agent","input","ecs","@version","log"]
        }
   }
}

output {

    if "aa" in [tags] {
                elasticsearch {
                        codec => "json"
                        hosts => "192.168.102.179:9200"
                        index => "aa"
                }
    } else if "bb" in [tags]{
                elasticsearch {
                        codec => "json"
                        hosts => "192.168.102.179:9200"
                        index => "bb-%{index_month}"
                }

    } else if "cc" in [tags] {
        elasticsearch {
            codec => "json"
            hosts => "192.168.102.179:9200"
            index => "cc_log"
        }
    }

else if "recommend" in [tags] {
        kafka {
            codec => json
            topic_id => "bbb"
            bootstrap_servers => "192.168.102.179:9092"
        }
    }else if "api" in [tags] {
     kafka {
            codec => json
            topic_id => "aaa"
            bootstrap_servers => "192.168.102.179:9092"
        }
   }
}

pipeline配置

- pipeline.id: main
  path.config: "/data/logstash-7.5.1/config/conf.d/*.conf"

2.3启动

启动bin目录下的脚本

nohup ./bin/logstash "--path.settings" "./config"  &

3.kibana安装

3.1下载

wget https://artifacts.elastic.co/downloads/kibana/kibana-6.3.2-linux-x86_64.tar.gz

3.2 配置

elasticsearch.hosts: ["http://192.168.102.179:9200"]
server.host: "0.0.0.0"

3.3启动

nohup ./bin/kibana --allow-root &