1. logstash 启动运行
bin/logstash -f config/logstash_nginx_filebeat.conf
2. logstash_nginx_filebeat.conf 是编写进行配置
input {
beats {
# filebeat 链接端口
port => "5044"
# 链接超时
client_inactivity_timeout => 36000
}
}
filter {
ruby {
init => "@kname =['remote_addr','time_local','request_method','status']"
code => "new_event = LogStash::Event.new(Hash[@kname.zip(event.get('message').split('|'))])
new_event.remove('@timestamp')
event.append(new_event)
"
}
geoip {
source => "remote_addr"
target => "geoip"
database => "/opt/GeoLite2-City_20190716/GeoLite2-City.mmdb"
}
date {
locale => "en"
match => [ "time_local", "dd/MMM/yyyy:HH:mm:ss Z" ]
}
mutate {
convert => [
"status","integer"
]
}
}
output {
elasticsearch {
hosts => ["localhost:9200"]
#自定义索引名,通过这个索引在kibana里面搜索并创建查看信息
index => "logstash-nginx-%{+YYYY.MM.dd}"
}
}
logstash 重启后的信息:
Attempting to install template {:manage_template=>{"index_patterns"=>"logstash-*", "version"=>60001, "settings"=>{"index.refresh_interval"=>"5s", "number_of_shards"=>1}, "mappings"=>{"dynamic_templates"=>[{"message_field"=>{"path_match"=>"message", "match_mapping_type"=>"string", "mapping"=>{"type"=>"text", "norms"=>false}}}, {"string_fields"=>{"match"=>"*", "match_mapping_type"=>"string", "mapping"=>{"type"=>"text", "norms"=>false, "fields"=>{"keyword"=>{"type"=>"keyword", "ignore_above"=>256}}}}}], "properties"=>{"@timestamp"=>{"type"=>"date"}, "@version"=>{"type"=>"keyword"}, "geoip"=>{"dynamic"=>true, "properties"=>{"ip"=>{"type"=>"ip"}, "location"=>{"type"=>"geo_point"}, "latitude"=>{"type"=>"half_float"}, "longitude"=>{"type"=>"half_float"}}}}}}}
index => "logstash-nginx-%{+YYYY.MM.dd}" 对应 es 中添加的模板 template 的 index_patterns,默认状态配置;
index_patterns 对应logstash配置 output -> elasticsearch -> index;
如果 logstash配置中的index 修改,那么 es 中对就的模板也要进行修改;
logstash 存放到 clickhouse
input {
# 示例输入插件,如file, beats, redis等
stdin {}
}
filter {
# 如果需要的话,添加filter插件来解析和转换数据
}
output {
clickhouse {
hosts => ["clickhouse-server:8123"] # ClickHouse服务器的地址和端口
database => "logstash_db" # 数据库名
table => "logstash_table" # 表名
username => "default" # 用户名(如果需要)
password => "password" # 密码(如果需要)
columns => {
"@timestamp" => "timestamp",
"message" => "message"
# 添加更多列映射
}
}
}
4个插件说明:Logstash中的四大插件_logstash 插件-CSDN博客
Logstash 原理分析/配置文件详解 时间 日期 时区 ip 反斜杠 grok在线地址 类型转换_grok日志在线转换-CSDN博客