【记录】Telegraf-解析nginx日志

已于 2022-06-22 20:22:11 修改
阅读量4.5k 收藏 2
点赞数
分类专栏： 监控 文章标签： 云计算 后端 golang nginx
于 2022-04-19 22:05:37 首次发布
版权声明：本文为博主原创文章，遵循 CC 4.0 BY-SA 版权协议，转载请附上原文出处链接和本声明。
本文链接：https://blog.csdn.net/qq_32212587/article/details/124284147
版权

本文档只记录解析nginx access.log相关内容，关于telegraf后期整理一篇文档单独介绍。

1、Tail Input Plugin

Tail Input Plugin 是 Telegraf项目 众多Inputs插件之一。
Tail Input Plugin 监听一个日志文件并解析该文件的每个日志变更。
默认情况下，tail 插件的行为类似于以下 Unix tail 命令：

	tail -F --lines=0 myfile.log

Telegraf收集nginx access.log 相当于Unix tail命令：

	tail -F --lines=0 access.log

并使用grok格式来解析日志。
除此之外，Telegraf Tail Input Plugin支持多种 Data Format
细节的东西可以跟下源码，下面直接记录运行配置。

2、配置

(1) telegraf.conf

[agent]
  interval = "10s"
  round_interval = true
  metric_batch_size = 1000
  metric_buffer_limit = 10000
  collection_jitter = "0s"
  flush_interval = "10s"
  flush_jitter = "0s"
  precision = ""
  hostname = ""
  omit_hostname = false

[[inputs.tail]]
  files = ["/Users/enic/github/telegraf/testdata/access.log"]
#   from_beginning = false
  from_beginning = true
  grok_patterns = ["%{COMMON_LOG_FORMAT}"]
  name_override = "nginx_access_log"
  grok_custom_pattern_files = []
  grok_custom_patterns = '''
      NGINX_ACCESS_LOG %{IP:remote_addr} - (-|%{WORD:remote_user}) \[%{HTTPDATE:time_local}\] %{QS:request} %{NUMBER:status:int} %{NUMBER:body_bytes_sent:int} %{QS:referrer} %{QS:agent} %{IPORHOST:xforwardedfor} %{IPORHOST:host} %{QS:upstream_addr} (-|%{NUMBER:upstream_status:int}) %{BASE10NUM:request_time:float} (-|%{BASE10NUM:upstream_response_time:float}) %{QS:cookie_uin} %{QS:cookie_luin}
  '''
  grok_timezone = "Local"
  data_format = "grok"

[[outputs.file]]
  files = ["/Users/enic/github/telegraf/testdata/outputs.log"]

(2)access.log

10.10.10.10 - - [19/Apr/2022:03:17:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:17:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:17:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:17:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:17:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:17:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:17:44 +0800] "POST /qps HTTP/1.1" 504 3693 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:17:54 +0800] "POST /latency HTTP/1.1" 504 3693 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:18:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:18:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:18:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:18:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:18:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:18:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.100 - - [19/Apr/2022:03:18:37 +0800] "GET / HTTP/1.1" 200 1839 "-" "Expanse, a Palo Alto Networks company, searches across the global IPv4 space multiple times per day to identify customers' presences on the Internet. If you would like to be excluded from our scans, please send IP addresses/domains to: scaninfo@paloaltonetworks.com" "-"
10.10.10.10 - - [19/Apr/2022:03:18:44 +0800] "POST /qps HTTP/1.1" 504 3693 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:18:54 +0800] "POST /latency HTTP/1.1" 504 3693 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:19:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:19:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:19:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:19:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:18:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:18:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:18:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:18:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:19:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:19:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:19:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:19:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:18:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:18:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:18:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:18:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:18:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:18:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:18:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:18:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"
10.10.10.10 - - [19/Apr/2022:03:18:34 +0800] "POST /user/login HTTP/1.1" 400 63 "-" "Go-http-client/1.1" "-"

3、操作

cd telegraf
make
./telegraf --config=/Users/enic/github/telegraf/testdata/telegraf.conf

4、output展示

nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST request="/user/login",http_version=1.1,ident="-",auth="-",resp_bytes=63i,client_ip="10.10.10.10" 1650309454000000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST request="/user/login",resp_bytes=63i,client_ip="10.10.10.10",ident="-",auth="-",http_version=1.1 1650309454001000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST ident="-",request="/user/login",auth="-",http_version=1.1,resp_bytes=63i,client_ip="10.10.10.10" 1650309454002000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST http_version=1.1,client_ip="10.10.10.10",ident="-",resp_bytes=63i,auth="-",request="/user/login" 1650309454003000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST client_ip="10.10.10.10",auth="-",ident="-",request="/user/login",http_version=1.1,resp_bytes=63i 1650309454004000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST ident="-",auth="-",request="/user/login",resp_bytes=63i,client_ip="10.10.10.10",http_version=1.1 1650309454005000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=504,verb=POST auth="-",request="/qps",http_version=1.1,resp_bytes=3693i,client_ip="10.10.10.10",ident="-" 1650309464000000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=504,verb=POST ident="-",auth="-",request="/latency",http_version=1.1,resp_bytes=3693i,client_ip="10.10.10.10" 1650309474000000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST auth="-",request="/user/login",resp_bytes=63i,client_ip="10.10.10.10",http_version=1.1,ident="-" 1650309514000000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST ident="-",auth="-",request="/user/login",client_ip="10.10.10.10",resp_bytes=63i,http_version=1.1 1650309514001000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST resp_bytes=63i,client_ip="10.10.10.10",ident="-",auth="-",request="/user/login",http_version=1.1 1650309514002000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST request="/user/login",resp_bytes=63i,ident="-",auth="-",http_version=1.1,client_ip="10.10.10.10" 1650309514003000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST client_ip="10.10.10.10",auth="-",ident="-",request="/user/login",http_version=1.1,resp_bytes=63i 1650309514004000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST resp_bytes=63i,ident="-",request="/user/login",client_ip="10.10.10.10",auth="-",http_version=1.1 1650309514005000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=200,verb=GET ident="-",request="/",resp_bytes=1839i,auth="-",http_version=1.1,client_ip="10.10.10.100" 1650309517000000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=504,verb=POST request="/qps",auth="-",ident="-",http_version=1.1,resp_bytes=3693i,client_ip="10.10.10.10" 1650309524000000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=504,verb=POST resp_bytes=3693i,auth="-",request="/latency",http_version=1.1,client_ip="10.10.10.10",ident="-" 1650309534000000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST http_version=1.1,resp_bytes=63i,auth="-",client_ip="10.10.10.10",ident="-",request="/user/login" 1650309574000000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST ident="-",auth="-",resp_bytes=63i,client_ip="10.10.10.10",request="/user/login",http_version=1.1 1650309574001000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST ident="-",request="/user/login",auth="-",http_version=1.1,resp_bytes=63i,client_ip="10.10.10.10" 1650309574002000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST auth="-",request="/user/login",client_ip="10.10.10.10",ident="-",http_version=1.1,resp_bytes=63i 1650309574003000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST http_version=1.1,resp_bytes=63i,client_ip="10.10.10.10",auth="-",request="/user/login",ident="-" 1650309514000000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST ident="-",auth="-",client_ip="10.10.10.10",request="/user/login",http_version=1.1,resp_bytes=63i 1650309514001000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST http_version=1.1,client_ip="10.10.10.10",request="/user/login",resp_bytes=63i,auth="-",ident="-" 1650309514002000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST resp_bytes=63i,ident="-",request="/user/login",http_version=1.1,client_ip="10.10.10.10",auth="-" 1650309514003000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST resp_bytes=63i,client_ip="10.10.10.10",ident="-",auth="-",request="/user/login",http_version=1.1 1650309574000000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST client_ip="10.10.10.10",ident="-",auth="-",resp_bytes=63i,request="/user/login",http_version=1.1 1650309574001000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST client_ip="10.10.10.10",resp_bytes=63i,ident="-",auth="-",request="/user/login",http_version=1.1 1650309574002000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST resp_bytes=63i,client_ip="10.10.10.10",ident="-",auth="-",request="/user/login",http_version=1.1 1650309574003000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST request="/user/login",resp_bytes=63i,client_ip="10.10.10.10",auth="-",http_version=1.1,ident="-" 1650309514000000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST auth="-",http_version=1.1,resp_bytes=63i,client_ip="10.10.10.10",request="/user/login",ident="-" 1650309514001000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST client_ip="10.10.10.10",auth="-",resp_bytes=63i,ident="-",request="/user/login",http_version=1.1 1650309514002000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST auth="-",resp_bytes=63i,request="/user/login",http_version=1.1,client_ip="10.10.10.10",ident="-" 1650309514003000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST request="/user/login",resp_bytes=63i,client_ip="10.10.10.10",ident="-",http_version=1.1,auth="-" 1650309514004000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST request="/user/login",client_ip="10.10.10.10",http_version=1.1,resp_bytes=63i,ident="-",auth="-" 1650309514005000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST request="/user/login",http_version=1.1,resp_bytes=63i,client_ip="10.10.10.10",ident="-",auth="-" 1650309514006000000
nginx_access_log,host=enicMBP,path=/Users/enic/github/telegraf/testdata/access.log,resp_code=400,verb=POST ident="-",request="/user/login",resp_bytes=63i,client_ip="10.10.10.10",auth="-",http_version=1.1 1650309514007000000
评论 2
添加红包

请填写红包祝福语或标题

个

红包个数最小为10个

元

红包金额最低5元

当前余额3.43元 前往充值 >
需支付：10.00元
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包

打赏作者

dnice

你的鼓励将是我创作的最大动力

¥1 ¥2 ¥4 ¥6 ¥10 ¥20
扫码支付：¥1
获取中
扫码支付

您的余额不足，请更换扫码支付或充值

打赏作者

实付元
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明：

1.余额是钱包充值的虚拟货币，按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载，可以购买VIP、付费专栏及课程。

余额充值