logstash常用参数
1 path
是必须的选项,每一个file配置,都至少有一个path
2 exclude
是不想监听的文件,logstash会自动忽略该文件的监听。配置的规则与path类似,支持字符串或者数组,但是要求必须是绝对路径。
3 start_position
是监听的位置,默认是end,即一个文件如果没有记录它的读取信息,则从文件的末尾开始读取,也就是说,仅仅读取新添加的内容。对于一些更新的日志类型的监听,通常直接使用end就可以了;相反,beginning就会从一个文件的头开始读取。但是如果记录过文件的读取信息,这个配置也就失去作用了。
4 sincedb_path
这个选项配置了默认的读取文件信息记录在哪个文件中,默认是按照文件的inode等信息自动生成。其中记录了inode、主设备号、次设备号以及读取的位置。因此,如果一个文件仅仅是重命名,那么它的inode以及其他信息就不会改变,因此也不会重新读取文件的任何信息。类似的,如果复制了一个文件,就相当于创建了一个新的inode,如果监听的是一个目录,就会读取该文件的所有信息。
收集单个系统日志并输出至文件
[root@elk-node01 config]# cat system-log.conf
input {
file {
type => "meassage-log"
path => "/var/log/messages"
start_position => "beginning" #"第一次从头收集,之后从新添加的日志收集"
}
file {
type => "secure-log"
path => "/var/log/secure"
start_position => "beginning"
}
}
output {
file {
path => "/tmp/%{type}.%{+yyyy.MM.dd}"
}
}
语法检测
../bin/logstash -f system-log.conf -t
运行查看结果
查看/tmp下面的文件即可
收集nginx日志和系统日志写入到elasticsearch
修改nginx日志格式为json格式 方便查看和Kibana 展示
log_format access_json '{"@timestamp":"$time_iso8601",'
'"host":"$server_addr",'
'"clientip":"$remote_addr",'
'"size":$body_bytes_sent,'
'"responsetime":$request_time,'
'"upstreamtime":"$upstream_response_time",'
'"upstreamhost":"$upstream_addr",'
'"http_host":"$host",'
'"url":"$uri",'
'"domain":"$host",'
'"xff":"$http_x_forwarded_for",'
'"referer":"$http_referer",'
'"status":"$status"}';
access_log /data/wwwlogs/access_nginx.log access_json;
[root@elk-node01 config]# cat system-log.yml
input {
file {
type => "system-message"
path => "/var/log/mess ages"
start_position => "beginning"
}
file {
type => "system-secure"
path => "/var/log/secure"
start_position => "beginning"
}
file {
type => "nginx-access"
path => "/data/wwwlogs/access_nginx.log"
start_position => "beginning"
codec => json
}
}
output {
if[type] == "nginx-access" {
elasticsearch {
index => "nginx-access-%{+YYYY.MM.dd}"
hosts => ["192.168.1.252:9200"]
}
}
if[type] == "system-message" {
elasticsearch {
index => "system-message-%{+YYYY.MM.dd}"
hosts => ["192.168.1.252:9200"]
}
}
if[type] == "system-secure" {
elasticsearch {
index => "system-secure-%{+YYYY.MM.dd}"
hosts => ["192.168.1.252:9200"]
}
}
}
运行访问nginx生成日志
[root@elk-node01 config]# ab -c 100 -n 100 http://192.168.1.252/
tomcat日志收集
修改server.xml日志格式
[root@elk-node01 apache-tomcat-8.5.39]# cat conf/server.xml
<Valve className="org.apache.catalina.valves.AccessLogValve" directory="logs"
prefix="tomcat_access_log" suffix=".log"
pattern="{"clientip":"%h","ClientUser":"%l","authenticated":"%u","AccessTime":"%t","method":"%r","status":"%s","SendBytes":"%b","Query?string":"%q","partner":"%{Referer}i","AgentVersion":"%{User-Agent}i"}" />
启动tomcat
[root@elk-node01 apache-tomcat-8.5.39] ./bin/startup.sh
查看日志格式
[root@elk-node01 apache-tomcat-8.5.39]# cat logs/tomcat_access_log.2019-04-03.log
{"clientip":"192.168.1.55","ClientUser":"-","authenticated":"-","AccessTime":"[03/Apr/2019:13:48:05 +0800]","method":"GET / HTTP/1.1","status":"200","SendBytes":"11286","Query?string":"","partner":"-","AgentVersion":"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:66.0) Gecko/20100101 Firefox/66.0"}
{"clientip":"192.168.1.55","ClientUser":"-","authenticated":"-","AccessTime":"[03/Apr/2019:13:48:05 +0800]","method":"GET /tomcat.css HTTP/1.1","status":"200","SendBytes":"5581","Query?string":"","partner":"http://192.168.1.252:8080/","AgentVersion":"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:66.0) Gecko/20100101 Firefox/66.0"}
{"clientip":"192.168.1.55","ClientUser":"-","authenticated":"-","AccessTime":"[03/Apr/2019:13:48:05 +0800]","method":"GET /tomcat.png HTTP/1.1","status":"200","SendBytes":"5103","Query?string":"","partner":"http://192.168.1.252:8080/","AgentVersion":"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:66.0) Gecko/20100101 Firefox/66.0"}
{"clientip":"192.168.1.55","ClientUser":"-","authenticated":"-","AccessTime":"[03/Apr/2019:13:48:05 +0800]","method":"GET /favicon.ico HTTP/1.1","status":"200","SendBytes":"21630","Query?string":"","partner":"-","AgentVersion":"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:66.0) Gecko/20100101 Firefox/66.0"}
写入elasticsearch
input {
file {
type => "tmcat-8080"
path => "/data/elk-services/logstash/config/apache-tomcat-8.5.39/logs/tomcat_access_log.2019-04-03.log"
start_position => "beginning"
codec => "json"
}
}
output{
if[type] == "tmcat-8080" {
elasticsearch {
index => "tomcat-8080-%{+YYYY.MM.dd}"
hosts => [ "192.168.1.252:9200"]
}
}
启动logstash
[root@elk-node01 logstash]# ./bin/logstash -f config/system-log.yml
检测数据
kibana创建tomcat-access索引
查看数据
多访问几次tomcat 来生成数据
转载于:https://blog.51cto.com/9025736/2373483