kibana根据es索引创建坐标地图geo-ip(不使用logstash)
filebeat配置文件
filebeat.inputs:
- type: log
enabled: true
paths:
- /var/log/nginx/access.log
json.keys_under_root: true
json.overwrite_keys: true
tags: ["access","nginx"]
filebeat.config.modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: false
setup.template.enabled: false
setup.template.settings:
index.number_of_shards: 3
setup.template.name: "filebeat"
setup.template.fields: "fields.yml"
setup.template.overwrite: false
setup.kibana:
output.elasticsearch:
hosts: ["183.129.255.169:9200"]
index: "logstash-%{+yyyy.MM.dd}"
#json.keys_under_root: true
#json.overwrite_keys: true
processors:
- add_host_metadata: ~
- add_cloud_metadata: ~
直接创建可视化坐标地图 会报如下错误 ，找不到类型为geo_point的字段
由于我这边是使用的filebeat+es+kibana，版本均为6.5.4 没有用到logstash。所以直接在es中对该索引进行操作。需要将es中location字段转换成geo_point类型。因为es mapping创建好之后就不能修改，所以这边只能采用新建一个修改过字段的索引并创建一个别名的方式。(我这边索引名称是logstash开头的是因为之前看到说kibana只支持logstash开头的geo_ip，没有进行验证。就直接这么创建，避免踩坑了)
首先查看原索引的类型 可以看到location的类型是text
创建一个新的索引 其中location字段进行修改
复制索引的内容到新的索引中
可以看到索引已经创建成功
删除旧的索引logstash-2019.01.17 并设置新索引的别名为logstash-2019.01.17
可以看到新索引以及别名
值得一提的是，创建完别名之后 需要重启filebeat，否则新的日志搜集不到，会报如下的错误
重启filebeat之后创建kibana索引
如果想要每天都生成，需要写一个重复上面修改索引字段的程定时任务。我是每天早上9点执行,因为东8区时间的问题
如果生成的index还是提示type不对 重启一下kibana。
#时间年月日小时分钟
Now=$(date +'%Y.%m.%d')
echo $Now
new_index_name=logstash-aliases-$Now
echo $new_index_name
old_index_name=logstash-$Now
echo $old_index_name
curl -H "Content-Type: application/json" -XPUT "http://10.6.11.176:9200/$new_index_name" -d '
{"mappings" : {
"doc" : {
"properties" : {
"@timestamp" : {
"type" : "date"
},
"beat" : {
"properties" : {
"hostname" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"name" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"version" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"body_bytes_sent" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"fields" : {
"properties" : {
"log_source" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"geoip_city" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"geoip_country_name" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"host" : {
"properties" : {
"architecture" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"containerized" : {
"type" : "boolean"
},
"id" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"name" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"os" : {
"properties" : {
"codename" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"family" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"platform" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"version" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
}
}
},
"http_user_agent" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"input" : {
"properties" : {
"type" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"json" : {
"type" : "object"
},
"location" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"message" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"meta" : {
"properties" : {
"cloud" : {
"properties" : {
"availability_zone" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"instance_id" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"provider" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"region" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
}
}
},
"offset" : {
"type" : "long"
},
"prospector" : {
"properties" : {
"type" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"remote_addr" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"request" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"request_method" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"request_time" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"source" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"status" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"tags" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"time" : {
"type" : "date"
},
"uri" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
}
}
}
}
'
#复制索引
curl -XPOST "http://10.6.11.176:9200/_reindex" -H 'Content-Type: application/json' -d'
{
"source": {
"index": "'$old_index_name'"
},
"dest": {
"index": "'$new_index_name'"
}
}'
#删除索引并创建别名
curl -XDELETE 10.6.11.176:9200/$old_index_name?pretty
curl -XPOST "http://10.6.11.176:9200/_aliases" -H 'Content-Type: application/json' -d'
{
"actions": [
{
"add": {
"index": "'$new_index_name'",
"alias": "'$old_index_name'"
}
}
]
}'
#重启filebeat
pkill filebeat
nohup /tmp/filebeat-6.5.4-linux-x86_64/filebeat -e -c /tmp/filebeat-6.5.4-linux-x86_64/filebeat.yml &
401
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
