项目架构图
Logstash 配置管理
安装logstash
购买弹性云服务器 4核8G
安装部署
[root@logstash ~]# vim /etc/hosts
192.168.1.21 es-0001
192.168.1.22 es-0002
192.168.1.23 es-0003
192.168.1.24 es-0004
192.168.1.25 es-0005
192.168.1.27 logstash
[root@logstash ~]# dnf install -y logstash
[root@logstash ~]# ln -s /etc/logstash /usr/share/logstash/config
软链接。将两个文件关联起来,logstash默认没有 配置文件,需要手写
最简单的配置
配置文件在/etc/logstash/conf.d/目录下新建 .conf写配置文件
[root@logstash ~]# vim /etc/logstash/conf.d/my.conf
input {
stdin {}
}
filter{
}
output{
stdout{}
}
[root@logstash ~]# /usr/share/logstash/bin/logstash
启动服务,第一次启动时间比较长,调用java虚拟机,启动后,会从键盘获取数据
[root@logstash ~]# /usr/share/logstash/bin/logstash-plugin list
查看众多的插件
插件与调试格式
json格式字符串: {"a":"1", "b":"2", "c":"3"}
[root@logstash ~]# vim /etc/logstash/conf.d/my.conf
input {
stdin { codec => "json" }
}
filter{
}
output{
stdout{ codec => "rubydebug" }
}
[root@logstash ~]# /usr/share/logstash/bin/logstash
- json格式字符串: 输入 {"a":"1", "b":"2", "c":"3"} 回车
按照rebydebug格式输出
这两个用的最多
input 模块
file 插件
file插件基本配置
[root@logstash ~]# touch /tmp/{a,b}.log
[root@logstash ~]# echo 'string 01' >>/tmp/a.log
[root@logstash ~]# echo 'string 02' >>/tmp/a.log
[root@logstash ~]# echo 'string 03' >>/tmp/a.log
[root@logstash ~]# vim /etc/logstash/conf.d/my.conf
input {
file {
path => ["/tmp/a.log", "/tmp/b.log"]
}
}
# filter { 不做任何修改 }
# output { 不做任何修改 }
# 启动程序,等待数据输出
[root@logstash ~]# /usr/share/logstash/bin/logstash
#---------------------------------------------------
# 在另一个终端模拟写入日志
[root@logstash ~]# echo 'string 04' >>/tmp/b.log
[root@logstash ~]# echo 'string 05' >>/tmp/a.log
默认logstate读取文件新内容,标签没有记录过的数据都是新的
file插件高级配置
# 删除默认书签文件
[root@logstash ~]# rm -rf /var/lib/logstash/plugins/inputs/file/.sincedb_*
[root@logstash ~]# cat /tmp/{a.log,b.log} >/tmp/c.log
[root@logstash ~]# vim /etc/logstash/conf.d/my.conf
input {
file {
path => ["/tmp/c.log"]
start_position => "beginning"
sincedb_path => "/var/lib/logstash/sincedb"
}
}
# filter { 不做任何修改 }
# output { 不做任何修改 }
[root@logstash ~]# /usr/share/logstash/bin/logstash
filter 模块
grok 插件
正则表达式分组匹配格式: (?<名字>正则表达式)
正则表达式宏调用格式: %{宏名称:名字}
宏文件路径 :
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-patterns-core-4.3.4/patterns
准备测试数据
# 从 web 服务器查找一条日志写入到日志文件
[root@logstash ~]# echo '60.26.217.109 - admin [13/Jan/2023:14:31:52 +0800] "GET /es/ HTTP/1.1" 200 148209 "http://127.70.79.1/es/" "curl/7.61.1"' >/tmp/c.log
# 调试技巧:设置路径为 /dev/null 可以多次反复测试
[root@logstash ~]# vim /etc/logstash/conf.d/my.conf
input {
file {
path => ["/tmp/c.log"]
start_position => "beginning"
sincedb_path => "/dev/null"
}
}
# filter { 不做任何修改 }
# output { 不做任何修改 }
[root@logstash ~]# /usr/share/logstash/bin/logstash
匹配IP地址测试
[root@logstash ~]# vim /etc/logstash/conf.d/my.conf
# input { 不做任何修改 }
filter {
grok {
match => { "message" => "(?<userIP>((25[0-5]|2[0-4]\d|1?\d?\d)\.){3}(25[0-5]|2[0-4]\d|1?\d?\d))" }
}
grok {
match => { "message" => "%{IP:clientIP}" }
}
}
# output { 不做任何修改 }
[root@logstash ~]# /usr/share/logstash/bin/logstash
使用宏格式化日志
[root@logstash ~]# vim /etc/logstash/conf.d/my.conf
input {
file {
path => ["/tmp/c.log"]
start_position => "beginning"
sincedb_path => "/var/lib/logstash/sincedb"
}
}
filter{
grok {
match => { "message" => "%{HTTPD_COMBINEDLOG}" }
remove_field => ["message"]
}
}
output{
stdout{ codec => "rubydebug" }
}
[root@logstash ~]# /usr/share/logstash/bin/logstash
output 模块
elasticsearch 插件
[root@logstash ~]# vim /etc/logstash/conf.d/my.conf
input {
file {
path => ["/tmp/c.log"]
start_position => "beginning"
sincedb_path => "/dev/null"
}
}
filter{
grok {
match => { "message" => "%{HTTPD_COMBINEDLOG}" }
remove_field => ["message"]
}
}
output{
stdout{ codec => "rubydebug" }
elasticsearch {
hosts => ["es-0002:9200","es-0003:9200"]
index => "weblog2-%{+YYYY.MM.dd}"
}
}
[root@logstash ~]# /usr/share/logstash/bin/logstash
将日志文件写入elastsearch 集群里面,在web页面访问
- 访问页面,查看 Head 插件,验证数据写入 Elasticsearch 成功
- http://124.70.1.159:8080/es-head/