使用logstash读取CSV数据导入Elasticsearch,并配置模板映射。

input {
  file {
    path => ["/root/opt/logstash-6.3.0/bin/logstash_csv/51n.csv"]  
    start_position => "beginning"
    add_field => {"DATATYPE"=>"PT"}  
    add_field => {"LAYER"=>"51csv"}  
  }
}
filter {
  csv {
    separator => ","
    columns => ["DZZYID","DZZYMC","DZ","YHFL","X","Y","LY","DHHM","FLDM","ZZJGDM","LX","GXSJ","FL"]
    
  }
  mutate {
    convert => {
      "DZZYID" => "string"
      "DZZYMC" => "string"
      "DZ" => "string"
      "YHFL" => "string"
      "X" => "string"
      "Y" => "string"
      "LY" => "string"
      "DHHM" => "string"
      "FLDM" => "string"
      "ZZJGDM" => "string"
      "LX" => "string"
      "GXSJ" => "string"
      "FL" => "string"
    }
  }
  mutate{
    add_field => {"LOCATION" => "%{Y},%{X}"}
  }
  mutate {  
    remove_field =>["message"] 
    remove_field =>["host"] 
    remove_field =>["@timestamp"] 
    remove_field =>["@version"] 
    remove_field =>["path"] 
    }
 }

output {
  elasticsearch {
        hosts => ["172.25.16.154:9200", "172.25.16.147:9200", "172.25.16.148:9200"]
        index => "51csv"
        document_type => "doc"
        template => "/root/opt/logstash-6.3.0/bin/logstash_csv/51csv.json"
        manage_template => true
        template_name => "51csv"
        template_overwrite => true
  }
}

 

51csv.json

{
"template": "51csv",
"mappings": {
"doc": {
"properties": {
"DZZYID": {
"type": "keyword"
},
"LAYER": {
"type": "keyword"
},
"DZZYMC": {
"type": "text",
"term_vector": "with_positions_offsets",
"analyzer": "ik_max_word"
},

"X": {
"type": "keyword"
},
"Y": {
"type": "keyword"
},
"LOCATION": {
"type": "geo_point"
},
"DATATYPE": {
"type": "keyword"
}

}
}
}
}

 

  • 0
    点赞
  • 2
    收藏
    觉得还不错? 一键收藏
  • 0
    评论
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值