ELK日志分析平台2——logstash数据采集

logstash简介

logstash安装

[root@server4 /mnt] rpm -ivh jdk-8u171-linux-x64.rpm
[root@server4 /mnt] rpm -ivh logstash-7.6.1.rpm

标准输入到标准输出

[root@server4 ~] /usr/share/logstash/bin/logstash -e 'input { stdin { } } output { stdout {} }'

标准输入到文件

[root@server4 /usr/share/logstash/bin] cd /etc/logstash/conf.d/
[root@server4 /etc/logstash/conf.d] cat test.conf
input {
	stdin {}
}

output {
 file {
   path => "/tmp/testfile" #输出的文件路径
   codec => line { format => "custom format: %{message}"} #定制数据格式
 }
}

[root@server4 /etc/logstash/conf.d] /usr/share/logstash/bin/logstash -f test.conf #指定配置文件运行

标准输入到es主机

[root@server4 /etc/logstash/conf.d] cat es.conf
input {
	stdin {}
}

output {
	stdout {}

	elasticsearch {
		hosts => ["172.25.51.3:9200"] #输出到的ES主机与端口
		index => "logstash-%{+yyyy.MM.dd}" #定制索引名称
	}
}
[root@server4 /etc/logstash/conf.d] /usr/share/logstash/bin/logstash -f es.conf

查看ES主机

文件内容输出到es主机

[root@server4 /etc/logstash/conf.d] cat eslog.conf
input {
        file {
		path => "/var/log/messages"
		start_position => "beginning"
	}
}
output {
	stdout {}
        elasticsearch {
                hosts => "172.25.51.3:9200"
                index => "syslog-%{+YYYY.MM.dd}"
        }
}
[root@server4 /etc/logstash/conf.d] /usr/share/logstash/bin/logstash -f eslog.conf

Syslog输入插件

[root@server4 /etc/logstash/conf.d] cat syslog.conf 
input {
        syslog {
                type => "rsyslog"
                port => 514
        }
}
output {
        stdout { }
}
[root@server4 /etc/logstash/conf.d] /usr/share/logstash/bin/logstash -f syslog.conf

server2、3：

[root@server3 ~] vim /etc/rsyslog.conf 
[root@server3 ~] systemctl restart rsyslog.service

grok过滤插件

[root@server4 /etc/logstash/conf.d] cat grok.conf
input {
	file {
		path => "/var/log/httpd/access_log"
		start_position => "beginning"
	}

}

filter {
	grok {
	match => { "message" => "%{HTTPD_COMBINEDLOG}" }
	}
}

output {
	stdout {}

	elasticsearch {
		hosts => ["172.25.51.1:9200"]
		index => "apachelog-%{+yyyy.MM.dd}"
	}
	
}

[root@server4 /etc/logstash/conf.d] systemctl restart start httpd
[root@server4 /etc/logstash/conf.d] systemctl start httpd
[root@server4 /etc/logstash/conf.d] cd /var/www/html/
[root@server4 /var/www/html] echo www.westos.org > index.html
curl 172.25.51.4 #真机访问
www.westos.org
[root@server4 /var/www/html] cat /var/log/httpd/access_log 
172.25.51.250 - - [14/Aug/2021:14:41:30 +0800] "GET / HTTP/1.1" 200 15 "-" "curl/7.61.1"

ab -n 50 -c1 http://172.25.51.4/index.html