使用fluent-bit把nginx日志同步到skywalking分析展示指标到仪表板
下载安装fluent-bit
你好!参考官网文档(官网)安装好以后,需要修改fluent-bit.conf配置文件
[SERVICE]
Flush 5
Daemon Off
Log_Level info
[INPUT]
Name tail
Path C:\user Files\nginx-1.24.0\logs\access.log
[FILTER]
Name lua
Match *
Script fluent-bit-script.lua
Call rewrite_body
[OUTPUT]
Name stdout
Match *
Format json
[OUTPUT]
Name http
Match *
Host 127.0.0.1
Port 12800
URI /v3/logs
Format json
在配置文件同目录创建 fluent-bit-script.lua文件,内容如下
function rewrite_body(tag, timestamp, record)
local newRecord = {}
newRecord["body"] = { json = { json = record.log } }
newRecord["service"] = "nginx::nginx"
newRecord["serviceInstance"] = "127.0.0.1"
return 1, timestamp, newRecord
end
然后启动fluent-bit 并且指定配置文件
配置nginx配置文件
修改nginx.conf文件中的日志打印格式,修改以后重启nginx查询日志内容是否已修改
http {
include mime.types;
default_type application/octet-stream;
log_format main '{"remote_addr": "$remote_addr",'
'"remote_user": "$remote_user",'
'"request": "$request",'
'"time": "$time_iso8601",'
'"status": "$status",'
'"request_time":"$request_time",'
'"body_bytes_sent": "$body_bytes_sent",'
'"http_referer": "$http_referer",'
'"http_user_agent": "$http_user_agent",'
'"http_x_forwarded_for": "$http_x_forwarded_for"}';
access_log logs/access.log main;
sendfile on;
#tcp_nopush on;
#keepalive_timeout 0;
keepalive_timeout 65;
#gzip on;
map $http_upgrade $connection_upgrade {
default upgrade;
'' close;
}
upstream grafana {
server 192.168.176.128:3000;
}
server {
listen 80;
server_name localhost;
#charset koi8-r;
#access_log logs/host.access.log main;
location / {
proxy_set_header Host $http_host;
proxy_pass http://grafana;
}
location /api/live {
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $connection_upgrade;
proxy_set_header Host $http_host;
proxy_pass http://grafana;
}
error_page 500 502 503 504 /50x.html;
location = /50x.html {
root html;
}
}
配置skywalking
1、首先配置lal规则的配置文件(config\lal\default.yaml),解析日志文件
rules:
- name: default
dsl: |
filter {
if (log.service == "nginx::nginx") {
json {
abortOnFailure true
}
extractor {
endpoint parsed.request as String
tag status: parsed.status
tag remote_addr: parsed.remote_addr
layer 'GENERAL'
metrics {
timestamp log.timestamp as Long
labels service: log.service, instance: log.serviceInstance
name "nginx_log_count"
value 1
}
metrics {
timestamp log.timestamp as Long
labels service: log.service, instance: log.serviceInstance
name "nginx_request_time"
value parsed.request_time as Double
}
metrics {
timestamp log.timestamp as Long
labels service: log.service, instance: log.serviceInstance
name "nginx_body_bytes_sent"
value parsed.body_bytes_sent as Long
}
metrics {
timestamp log.timestamp as Long
labels service: log.service, instance: log.serviceInstance, status: parsed.status
name "nginx_status_code"
value 1
}
}
}
sink {
}
}
2、新建mal规则的配置文件放在(config\log-mal-rules\)目录下 my-lal-mal-config.yaml
expSuffix: service(['service'], Layer.GENERAL)
metricPrefix: nginx
metricsRules:
- name: cpm
exp: nginx_log_count.sum(['service'])
- name: avg_request_time
exp: nginx_request_time.avg(['service'])
- name: body_bytes_sent_count
exp: nginx_body_bytes_sent.sum(['service'])
- name: status_code_count
exp: nginx_status_code.sum(['service','status'])
3、配置skywalking配置文件 application.yml 修改mal文件配置,找到此配置项,把default.malFiles配置改为新创建my-lal-mal-config文件名称
log-analyzer:
selector: ${SW_LOG_ANALYZER:default}
default:
lalFiles: ${SW_LOG_LAL_FILES:default}
malFiles: ${SW_LOG_MAL_FILES:my-lal-mal-config}
4、创建新的仪表板
[{"id":"a9e3e6cc-6e59-46bc-92a1-110a1a73dc60","configuration":{"children":[{"x":0,"y":0,"w":12,"h":16,"i":"0","type":"Widget","metricMode":"Expression","expressions":["nginx_cpm"],"typesOfMQE":["TIME_SERIES_VALUES"],"graph":{"type":"Line","step":false,"smooth":false,"showSymbol":true,"showXAxis":true,"showYAxis":true},"widget":{"name":"CPM","title":"CPM"}},{"x":12,"y":0,"w":12,"h":16,"i":"1","type":"Widget","metricMode":"Expression","expressions":["nginx_avg_request_time"],"typesOfMQE":["TIME_SERIES_VALUES"],"graph":{"type":"Line","step":false,"smooth":false,"showSymbol":true,"showXAxis":true,"showYAxis":true},"widget":{"title":"avg_request_time"}},{"x":0,"y":16,"w":12,"h":16,"i":"2","type":"Widget","metricMode":"Expression","expressions":["nginx_body_bytes_sent_count/1024"],"typesOfMQE":["TIME_SERIES_VALUES"],"graph":{"type":"Line","step":false,"smooth":false,"showSymbol":true,"showXAxis":true,"showYAxis":true},"widget":{"title":"body_bytes_sent_count"}},{"x":12,"y":16,"w":12,"h":16,"i":"3","type":"Widget","metricMode":"Expression","expressions":["nginx_status_code_count"],"typesOfMQE":["TIME_SERIES_VALUES"],"graph":{"type":"Line","step":false,"smooth":false,"showSymbol":true,"showXAxis":true,"showYAxis":true},"widget":{"title":"status_code_count"}}],"layer":"GENERAL","entity":"Service","name":"Nginx","isRoot":false}}]
5、效果图