在需要收集日志的proxy节点上配置filebeat,步骤如下:
1、安装filebeat,执行如下命令:
yum -y install filebeat
2、修改/etc/filebeat/filebeat.yml文件,内容如下:
filebeat.prospectors:
- input_type: log
paths:
- /var/log/swift/proxy-server.log
document_type: "swift-proxy"
fields:
logsource: hostname #修改为本机主机名
logtype: swift-proxy
logcluster: swift-cluster-01 #修改为本集群名称
fields_under_root: true
output.logstash:
hosts: ["192.168.25.31:10515"] #多个用逗号隔开
worker: 2
loadbalance: true
3、启动filebeat服务:
systemctl enable filebeat.service
systemctl start filebeat.service
配置logstash的filter:(安装请参考ELK部署)
vim /etc/logstash/conf.d/swift.conf
添加如下内容:
input {
beats {
host => "0.0.0.0"
port => 10515
}
}
filter {
if [fields][logtype] == "swift-proxy" {
if "ERROR" in [message] {
grok {
match => ["message", "(?<timestamp>[a-zA-Z]{,3} [0-9]{,2}) %{HOUR}:%{MINUTE}:%{SECOND} (?<hostname>swift[0-9]{,2}[aA-zZ]{2,}[0-9]+ proxy-server): (?<o_msg>.*)"]
}
date {
match => ["timestamp", "dd/MMM/yyyy:HH:mm:ss Z"]
timezone => "Etc/UTC"
}
}
else {
grok {
match => ["message", "(?<t_date>[a-zA-Z]{,3} [0-9]{,2}) %{HOUR}:%{MINUTE}:%{SECOND} (?<hostname>swift[0-9]{,2}[aA-zZ]{2,}[0-9]+ proxy-server): (%{IPV4:client_ip}|-) (%{IPV4:remote_ip}|-) (?<timestamp>[0-9]{,2}\/[a-zA-Z]{,3}\/[0-9]{,4}\/[0-9]{,2}\/[0-9]{,2}\/[0-9]{,2}) %{WORD:method} %{NOTSPACE:query_string} HTTP/(?<version>[0-9].[0-9]) (?<status>[0-9]{,3}) (?<o_msg>.* tx[a-zA-Z0-9]+-[a-zA-Z0-9]+) - %{BASE10NUM:request_time} (-|RL)"]
}
date {
match => ["timestamp", "dd/MMM/yyyy:HH:mm:ss Z" ]
timezone => "Etc/UTC"
}
}
}
}
output {
if [fields][logtype] == "swift-proxy" and [fields][logcluster] =="swift-cluster-01"{
elasticsearch {
hosts => ["192.168.25.30:9200"] #多个用逗号隔开["",""]
index => "swift-cluster-01-swift-proxy-%{+YYYY.MM.dd}"
}
}
else if [fields][logtype] == "swift-proxy" and [fields][logcluster] =="swift-cluster-01"{
elasticsearch {
hosts => ["192.168.25.30:9200"] #多个用逗号隔开
index => "swift-cluster-01-swift-proxy-%{+YYYY.MM.dd}"
}
}
else{
elasticsearch {
hosts => ["192.168.25.30:9200","192.168.25.31:9200","192.168.25.32:9200"] #多个用逗号隔开
index => "swift-proxy-%{+YYYY.MM.dd}"
}
}
}