logstash的配置

flow-es：

input {
  file {
    type => "flow"
    path => "/var/nginx_logs/*.log"
    discover_interval => 5
    start_position => "beginning" 
  }
}

output {
  if [type] == "flow" {
    elasticsearch {
      index => "flow-%{+YYYY.MM.dd}"
      hosts => ["172.16.0.14:9200", "172.16.0.15:9200", "172.16.0.16:9200"]
    }
  }   
}

flow-kafka：

input {
  file {
    path => "/var/nginx_logs/*.log"
    discover_interval => 5
    start_position => "beginning" 
  }
}

output {
	kafka {
	  topic_id => "accesslog"
	  codec => plain {
		format => "%{message}"
		charset => "UTF-8"
	  }
	  bootstrap_servers => "172.16.0.11:9092,172.16.0.12:9092,172.16.0.13:9092"
	}
}

gameserver-kafka：

input {
  file {
	codec => plain {
      charset => "GB2312"
    }
    path => "D:/MirServer/LogServer/BaseDir/*/*.txt"
    discover_interval => 30
    start_position => "beginning"
  }
}

output {
    kafka {
	  topic_id => "itcast"
	  codec => plain {
        format => "%{message}"
		charset => "GB2312"
      }
	  bootstrap_servers => "172.16.0.11:9092,172.16.0.12:9092,172.16.0.13:9092"
    }
}

kafka-es：

input {
  kafka {
    type => "level-one"
    auto_offset_reset => "smallest"
    codec => plain {
      charset => "GB2312"
    }
	group_id => "es"
	topic_id => "itcast"
	zk_connect => "172.16.0.11:2181,172.16.0.12:2181,172.16.0.13:2181"
  }
}

filter {
  mutate {
    split => { "message" => "	" }
      add_field => {
        "event_type" => "%{message[3]}"
        "current_map" => "%{message[4]}"
        "current_X" => "%{message[5]}"
        "current_y" => "%{message[6]}"
        "user" => "%{message[7]}"
        "item" => "%{message[8]}"
        "item_id" => "%{message[9]}"
        "current_time" => "%{message[12]}"
     }
     remove_field => [ "message" ]
  } 
}

output {
    elasticsearch {
      index => "level-one-%{+YYYY.MM.dd}"
	  codec => plain {
        charset => "GB2312"
      }
      hosts => ["172.16.0.14:9200", "172.16.0.15:9200", "172.16.0.16:9200"]
    } 
}