CentOS6.9安装Logstash

一、下载地址

官网：https://www.elastic.co/cn/downloads/logstash

百度云盘：

二、安装

tar zxvf logstash-6.2.1.tar.gz
mv logstash-6.2.1 logstash

配置文件（配置文件放哪个目录都可以，在启动Logstash时可以指定配置文件位置）

vi k_es.conf

input{
    kafka {
        bootstrap_servers => "10.10.6.225:9092"    #kafka服务器地址，不是zookeeper
    client_id => "test"
        auto_offset_reset => "latest"
        consumer_threads => 5
        decorate_events => true
        topics => ["test"]    #控制kafka哪个topic，可以多个用逗号分割
    codec => "json"        
    } 
}
filter{      
    json {
            source => "message"            
        remove_field=>["message","beat","@version"]    #删除没用的属性
        add_field =>["customize","自定义字段"]    #增加属性
    }
}  
output {
  elasticsearch {
        hosts => "localhost:9200"
        index => "logstash-nginxacclog-%{+YYYY.MM.dd}"    #放到elasticsearch哪个index中
    }    
}

启动（到bin目录下）

./logstash -f k_es.conf 

后台运行

nohup /usr/local/logstash/bin/logstash -f /usr/local/logstash/bin/k_es.conf -w 8 -b 1000 > /dev/null 2>&1 &

配置文件备份

input{
    kafka {
        bootstrap_servers => "10.100.2.210:9092"
        client_id => "nginxaccesslog"
        auto_offset_reset => "latest"
        consumer_threads => 5
        decorate_events => true
        topics => ["nginx_access_log"]
        codec => "json"
        type => "nginx_log"
    }
    kafka {
        bootstrap_servers => "10.100.2.210:9092"
        client_id => "database"
        auto_offset_reset => "latest"
        consumer_threads => 5
        decorate_events => true
        topics => ["t_resource_info","t_base_person","t_base_organization","t_base_student","t_base_parent","t_base_class"]
        codec => "json"
        type => "dsideal_db"
    }
     kafka {
        bootstrap_servers => "10.100.2.210:9092"
        client_id => "devops_real"
        auto_offset_reset => "latest"
        consumer_threads => 5
        decorate_events => true
        topics => ["devopsrealinfo"]
                codec => "json"
                type => "devopsrealinfo"
        }
    kafka {
        bootstrap_servers => "10.100.2.210:9092"
        client_id => "devops_base"
        auto_offset_reset => "latest"
        consumer_threads => 5
        decorate_events => true
        topics => ["devopsbaseinfo"]
                codec => "json"
                type => "devopsbaseinfo"
        }
}
filter{
    mutate {
        gsub => ["message", "\\x22", '"']
    }
    json {
        source => "message"
        remove_field=>["message","beat","@version"]
    }
    if [type] == "nginx_log" {
        geoip {
                source => "ip"
                target => "geoip"
                database => "/usr/local/GeoLite2-City.mmdb"
        }
    }
}
output {
    if [type] == "nginx_log" {
        elasticsearch {
            hosts => "10.100.2.210:9200"
            index => "nginx-access-log"
        }
    }
    if [type] == "dsideal_db" {
        elasticsearch {
            hosts => "10.100.2.210:9200"
            index => "%{table_name}"
            document_id => "%{id}"
        }
    }
    if [type] == "devopsbaseinfo" {
        elasticsearch {
                hosts => "localhost:9200"
                index => "devopsbaseinfo"
                document_id => "%{id}"
        }
    }
    if [type] == "devopsrealinfo" {
        elasticsearch {
                hosts => "localhost:9200"
                index => "devopsrealinfo"
        }
    }
}