Logstash分析APP日志

Logstash分析APP日志

#准备日志，此处我只是收集了一台服务器中的app日志，可以收集多台
#此处的jar文件在我的资源中免费下载
[root@web01 ~]# ls
 app-dashboard-1.0-SNAPSHOT.jar
[root@web01 ~]# java -jar app-dashboard-1.0-SNAPSHOT.jar &>/var/log/app.log
[root@web01 ~]# tail -f /var/log/app.log 
[INFO] 2021-07-04 22:20:06 [cn.oldxu.dashboard.Main] - DAU|1576|搜索|2021-07-04 08:18:01
[INFO] 2021-07-04 22:20:08 [cn.oldxu.dashboard.Main] - DAU|4206|加入购物车|2021-07-04 09:05:08
[INFO] 2021-07-04 22:20:09 [cn.oldxu.dashboard.Main] - DAU|8291|查看订单|2021-07-04 17:11:07
[INFO] 2021-07-04 22:20:11 [cn.oldxu.dashboard.Main] - DAU|7144|使用优惠券|2021-07-04 21:07:06

#先将日志输出到控制台，下一步看怎么转换
[root@logstash-node1 conf.d]# cat input_filebeat_app_output_es.conf 
input {
       http {
                port => 7474
   }
}

output {
       stdout {
                codec =>rubydebug
    }
}

#启动logstash
[root@logstash-node1 conf.d]# /usr/share/logstash/bin/logstash -f input_filebeat_app_output_es.conf -r

发送日志

#根据需求在改文件
#下面是我们将需要转换的信息转为json，将不需要的信息移除掉
[root@logstash-node1 conf.d]# cat input_filebeat_app_output_es.conf 
input {
       http {
                port => 7474
   }
}

filter {
         mutate {
                  split => {message => "|"}
                  add_field => {
                            "UserID" => "%{[message][1]}"
                            "Action" => "%{[message][2]}"
                            "Date" => "%{[message][3]}"
           }
       }

        date {
		match => ["Date","yyyy-MM-dd HH:mm:ss"]
		target => "@timestamp"
		timezone => "Asia/Chongqing"
	}
        mutate {
		#remove_field => ["message","Date"]
		add_field => { "target_index" => "logstash-app-%{+YYYY.MM.dd}" }
	}
  }

output {
       stdout {
                codec =>rubydebug
    }

}

发送消息

下面我们将这些信息输入到ES集群中

#filebeat配置
[root@web01 ~]# cat /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/app.log


output.logstash:
  hosts: ["10.0.0.151:5044"]
[root@web01 ~]# systemctl restart filebeat

#logstash配置
[root@logstash-node1 conf.d]# cat input_filebeat_app_output_es.conf
input {
        beats  {
                port => 5044
   }
}

filter {
         mutate {
                  split => {message => "|"}
                  add_field => {
                            "UserID" => "%{[message][1]}"
                            "Action" => "%{[message][2]}"
                            "Date" => "%{[message][3]}"
           }
       }

        date {
		match => ["Date","yyyy-MM-dd HH:mm:ss"]
		target => "@timestamp"
		timezone => "Asia/Chongqing"
	}
        mutate {
		#remove_field => ["message","Date"]
		add_field => { "target_index" => "logstash-app-%{+YYYY.MM.dd}" }
	}
  }

output {
       #这是输出到控制台，如果不用可以删除
       stdout {
                codec =>rubydebug
    }
       elasticsearch {
                 hosts => ["10.0.0.161:9200","10.0.0.162:9200","10.0.0.163:9200"]
                 index => "%{[target_index]}"
  }
}