1.下载并安装logstash
logstash-7.9.2.tar.gz
--创建目录
mkdir -p /esdb/logstash/app
mkdir -p /esdb/logstash/data
mkdir -p /esdb/logstash/logs
tar xvf logstash-7.9.2.tar.gz
[esadmin@oracle1 soft]$ cp -r logstash-7.9.2/* /esdb/logstash/app/
*/
2.logstash 配置文件
vim /esdb/logstash/app/config/logstash.yml
path.data: /esdb/logstash/data
path.logs: /esdb/logstash/logs
pipeline.workers: 2
pipeline.batch.size: 3000
pipeline.batch.delay: 5
queue.type: persisted
path.queue: /esdb/logstash/data
queue.page_capacity: 250mb
queue.max_bytes: 1024mb
queue.checkpoint.acks: 0
queue.checkpoint.writes: 0
queue.checkpoint.interval: 1000
3.jvm配置
--修改JVM参数。
vi /esdb/logstash/app/config/jvm.options
-server
-Xms512M
-Xmx512M
-XX:+DisableExplicitGC
-XX:+UseGlGC
-XX:G1ReservePercent=25
-XX:InitiatingHeapOccupancyPercent=30
--注释掉这些。
#-XX:+UseConcMarkSweepGC
#-XX:CMSInitiatingOccupancyFraction=75
#-XX:+UseCMSInitiatingOccupancyOnly
#如下是遇到报错时才加的
--add-opens java.base/sun.nio.ch=ALL-UNNAMED
--add-opens java.base/java.io=ALL-UNNAMED
--add-opens java.base/java.security=ALL-UNNAMED
4.配置logstash 读取tomcat的日志数据
--从头开始读取tomcat的日志信息。过滤处理后最终同步到es数据库集群。
vim /esdb/logstash/app/config/sspu-tomcat.conf
input {
file {
path => "/tomcat/tomcat8080/logs/tomcat_access_json.*.log"
tags => "tomcatlog"
start_position => "beginning"
stat_interval => "2"
codec => "json"
}
}
filter {
mutate {
gsub => ["message","\\x","\\\x"]
}
json {
source => "message"
}
mutate {
add_field => { "tomcat_ip" => "%{[@metadata][clientip]}" }
convert => ["[geoip][coordinates]", "float"]
remove_field => [ "host" ]
}
}
output {
elasticsearch {
hosts => ["192.168.1.7:9201", "192.168.1.7:9202","192.168.1.7:9203"]
index => "sspu-tomcat-%{+YYYY.MM.dd}"
user => elastic
password => esadmin
}
}
5.启动logstash
chmod 755 /esdb/logstash/app/config/sspu-tomcat.conf
nohup /esdb/logstash/app/bin/logstash -f /esdb/logstash/app/config/sspu-tomcat.conf &
6.Logstash启动报错。
[esadmin@oracle1 config]$ /esdb/logstash/app/bin/logstash -f /esdb/logstash/app/config/sspu-tomcat.conf &
[1] 60298
[esadmin@oracle1 config]$
[esadmin@oracle1 config]$ 2024-06-30T11:05:03.548+08:00 [main] WARN FilenoUtil : Native subprocess control requires open access to the JDK IO subsystem
Pass '--add-opens java.base/sun.nio.ch=ALL-UNNAMED --add-opens java.base/java.io=ALL-UNNAMED' to enable.
[ERROR] 2024-06-30 11:05:28.735 [main] Logstash - java.lang.IllegalStateException:
Logstash stopped processing because of an error: (LoadError) load error:
rubygems/security -- java.lang.reflect.InaccessibleObjectException:
Unable to make field private java.security.Provider
java.security.MessageDigest.provider accessible:
module java.base does not "opens java.security" to unnamed module @4920430f
[1]+ Exit 1 /esdb/logstash/app/bin/logstash -f /esdb/logstash/app/config/sspu-tomcat.conf
[esadmin@oracle1 config]$ [ERROR] 2024-06-30 11:11:26.725
[main] Logstash - java.lang.IllegalStateException:
Logstash stopped processing because of an error: (LoadError) load error:
rubygems/security -- java.lang.reflect.InaccessibleObjectException:
Unable to make field private java.security.Provider
java.security.MessageDigest.provider accessible:
module java.base does not "opens java.security" to unnamed module @3815c525
[esadmin@oracle1 config]$ vi jvm.options
--添加如下选项:启用未命名的模块。
--add-opens java.base/sun.nio.ch=ALL-UNNAMED
--add-opens java.base/java.io=ALL-UNNAMED
--add-opens java.base/java.security=ALL-UNNAMED
7.成功启动Logstash
[esadmin@oracle1 config]$ /esdb/logstash/app/bin/logstash -f /esdb/logstash/app/config/sspu-tomcat.conf &
[1] 60344
[2024-06-30T11:12:50,601][INFO ][logstash.inputs.file ][main] No sincedb_path set, generating one based on the "path" setting {:sincedb_path=>"/esdb/logstash/data/plugins/inputs/file/.sincedb_df1acf137331f4666e360b1b555da55d", :path=>["/tomcat/tomcat8080/logs/tomcat_access_json.*.log"]}
[2024-06-30T11:12:50,657][INFO ][logstash.javapipeline ][main] Pipeline started {"pipeline.id"=>"main"}
[2024-06-30T11:12:50,796][INFO ][filewatch.observingtail ][main][55443a7aea8e3d96e851294e44ab63120d0b358df1c3ed62930cc435db64cc63] START, creating Discoverer, Watch with file and sincedb collections
[2024-06-30T11:12:50,869][INFO ][logstash.agent ] Pipelines running {:count=>1, :running_pipelines=>[:main], :non_running_pipelines=>[]}
[2024-06-30T11:12:51,776][INFO ][logstash.agent ] Successfully started Logstash API endpoint {:port=>9600}
8.ES 收集的日志检查
kibana查询:
get sspu-tomcat-*
{
"sspu-tomcat-2024.06.30" : {
"aliases" : { },
"mappings" : {
"properties" : {
"@timestamp" : {
"type" : "date"
},
"@version" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"AccessTime" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"AgentVersion" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"ClientUser" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"Query?string" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"SendBytes" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"authenticated" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"clientip" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"method" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"partner" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"path" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"status" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"tags" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"tomcat_ip" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"settings" : {
"index" : {
"creation_date" : "1719717172674",
"number_of_shards" : "3",
"number_of_replicas" : "1",
"uuid" : "ECZRgaS8RZeHSLOKI0CuOw",
"version" : {
"created" : "7090299"
},
"provided_name" : "sspu-tomcat-2024.06.30"
}
}
}
}