日志样例
JAVA
固定格式
[2022-05-30T14:54:07.579+08:00] [INFO] [,] [cn.git.workflow.util.WorkFlowFactory] [ccms-test-17] [3.1.101.55] [workflow-server] [WorkFlowFactory.java,163,cn.git.workflow.util.WorkFlowFactory,getWebServiceProperties] [webService获取地址类型serviceType为[UAT],选择操作url地址为[http://6.1.14.86:9001/wfms/cxf/processWebService?wsdl]] ## ''
[2022-05-30T14:54:07.624+08:00] [INFO] [,] [cn.git.workflow.service.impl.WorkFlowServiceImpl] [ccms-test-17] [3.1.101.55] [workflow-server] [WorkFlowServiceImpl.java,414,cn.git.workflow.service.impl.WorkFlowServiceImpl,findProcessPic] [查看流程图发送参数: {"processId":"787169"}] ## ''
NGINX
access日志
直接配置nginx.conf, 配置为json格式,固定格式
log_format json escape=json
'{"@timestamp":"$time_iso8601",'
'"host":"$hostname",'
'"server_ip":"$server_addr",'
'"client_ip":"$remote_addr",'
'"xff":"$http_x_forwarded_for",'
'"domain":"$host",'
'"url":"$uri",'
'"referer":"$http_referer",'
'"args":"$args",'
'"upstreamtime":"$upstream_response_time",'
'"responsetime":"$request_time",'
'"request_method":"$request_method",'
'"status":"$status",'
'"size":"$body_bytes_sent",'
# '"request_body":"$request_body",'
'"request_length":"$request_length",'
'"protocol":"$server_protocol",'
'"upstreamhost":"$upstream_addr",'
'"upstreamtime":"$upstream_response_time",'
'"upstreamstatus":"$upstream_status",'
'"file_dir":"$request_filename",'
'"http_user_agent":"$http_user_agent"'
'}';
error日志
默认配置, 输出格式不固定
格式一
2022/05/30 14:48:45 [warn] 13#0: *87124 an upstream response is buffered to a te1/23/0000000231 while reading upstream, client: 3.2.26.199, server: 3.1.101.57, P/1.1", upstream: "http://3.1.101.56:11102/manage/common/syscodes/get", host: "3"
格式二
2022/05/30 10:37:03 [error] 13#0: *81714 connect() failed (111: Connection refused) while connecting to upstream, client: 3.2.26.154, server: 3.1.101.57, request: "GET /sysmanage/manage/organization/listps HTTP/1.1", upstream: "http://3.1.101.56:11102/manage/organization/listps", host: "3.1.101.57:8901", referrer: "http://3.1.101.57:8901/"
格式三
2022/05/30 15:12:33 [emerg] 1#0: unexpected end of file, expecting "}" in /usr/local/openresty/nginx/conf/nginx.conf:105
2022/05/30 22:41:12 [notice] 22802#0: signal process started
filebeat配置
###################### Filebeat Configuration Example #########################
filebeat.name: ccms-test-19
filebeat.idle_timeout: 5s
filebeat.spool_zie: 2048
#----------------------------------input form ccms servers--------------------------------#
filebeat.inputs:
- type: log
enabled: true
paths:
- /opt/ccms-auto-deploy/credit-interface-converter/*/target/logs/*.log
- /opt/ccms-auto-deploy/credit-gateway/target/logs/*.log
fields:
kafka_topic: topic-ccms-dev
filebeat-server: 3.1.101.57
fields_under_root: true
# filebeat 多行日志的处理
multiline.pattern: '^\['
multiline.negate: true
multiline.match: after
encoding: plain
tail_files: false
# 检测指定目录下文件更新时间
scan_frequency: 3s
# 每隔1s检测一下文件变化,如果连续检测2次之后文件还没有变化,下一次检测间隔时间变为5s
backoff: 1s
max_backoff: 5s
backoff_factor: 2
#----------------------------------input form nginx access_log--------------------------------#
- type: log
enabled: true
paths:
- /data/openresty-vts/nginx/logs/ccms-*.log
fields:
kafka_topic: topic-nginx-access
filebeat-server: 3.1.101.57
fields_under_root: true
encoding: plain
tail_files: false
json.keys_under_root: true
json.overwrite_keys: true
json.add_error_key: false
# 检测指定目录下文件更新时间
scan_frequency: 3s
# 每隔1s检测一下文件变化,如果连续检测2次之后文件还没有变化,下一次检测间隔时间变为5s
backoff: 1s
max_backoff: 5s
backoff_factor: 2
#----------------------------------input form nginx error_log--------------------------------#
- type: log
enabled: true
paths:
- /data/openresty-vts/nginx/logs/error.log
fields:
kafka_topic: topic-nginx-error
filebeat-server: 3.1.101.57
fields_under_root: true
encoding: plain
tail_files: false
# 检测指定目录下文件更新时间
scan_frequency: 3s
# 每隔1s检测一下文件变化,如果连续检测2次之后文件还没有变化,下一次检测间隔时间变为5s
backoff: 1s
max_backoff: 5s
backoff_factor: 2
#----------------------------------Kafka output--------------------------------#
output.kafka:
enabled: true
hosts: ['3.1.101.33:9092','3.1.101.34:9092','3.1.101.35:9092']
topic: '%{[kafka_topic]}'
logstash配置
JAVA
input {
beats {
port => 5044
}
}
input {
kafka {
topics_pattern => "topic-ccms-dev"
bootstrap_servers => "3.1.101.33:9092,3.1.101.34:9092,3.1.101.35:9092"
consumer_threads => 4
decorate_events => true
group_id => "kafka-elk-ccms"
add_field => {"logstash-server" => "3.1.101.33"}
}
}
filter {
json {
source => "message"
}
grok {
match => { "message" => "\[%{TIMESTAMP_ISO8601:currentDateTime}\] \[%{LOGLEVEL:level}\] \[%{DATA:traceInfo}\] \[%{NOTSPACE:class}\] \[%{DATA:hostName}\] \[%{IP:hostIp}\] \[%{DATA:applicationName}\] \[%{DATA:location}\] \[%{DATA:messageInfo}\] ## %{QUOTEDSTRING:throwable}" }
}
mutate{
enable_metric => "false"
remove_field => ["ecs","tags","input","agent","@version","log","port","host","message"]
}
date {
match => [ "currentDateTime", "ISO8601" ]
}
}
output {
elasticsearch {
hosts => ["3.1.101.33:9200","3.1.101.34:9200","3.1.101.35:9200"]
index => "index-ccms-dev_%{+YYY-MM-dd}"
sniffing => true
template_overwrite => true
}
}
NGINX
access日志
input {
kafka {
topics_pattern => "topic-nginx-access"
bootstrap_servers => "3.1.101.33:9092,3.1.101.34:9092,3.1.101.35:9092"
codec => "json"
consumer_threads => 4
decorate_events => true
group_id => "kafka-nginx-access"
add_field => {"logstash-server" => "3.1.101.33"}
}
}
filter {
geoip {
source => "client_ip"
target => "geoip"
add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ]
remove_field => [ "[geoip][latitude]", "[geoip][longitude]", "[geoip][country_code2]","[geoip][country_code3]", "[geoip][timezone]", "[geoip][continent_code]", "[geoip][dma_code]", "[geoip][region_code]" ]
}
mutate {
convert => [ "size", "integer" ]
convert => [ "status", "integer" ]
convert => [ "responsetime", "float" ]
convert => [ "upstreamtime", "float" ]
convert => [ "[geoip][coordinates]", "float" ]
# 过滤 filebeat 没用的字段,这里过滤的字段要考虑好输出到es的,否则过滤了就没法做判断
remove_field => [ "ecs","agent","host","cloud","@version","input","logs_type" ]
}
useragent {
source => "http_user_agent"
target => "ua"
# 过滤useragent没用的字段
remove_field => [ "[ua][minor]","[ua][major]","[ua][build]","[ua][patch]","[ua][os_minor]","[ua][os_major]" ]
}
}
output {
elasticsearch {
hosts => ["3.1.101.33:9200","3.1.101.34:9200","3.1.101.35:9200"]
index => "logstash-nginx-access_%{+YYY-MM-dd}"
sniffing => true
template_overwrite => true
}
}
这里是为了适应grafana 11190模板, 将ES数据展示到grafana中, 实际可根据需求调整
error日志
input {
kafka {
topics_pattern => "topic-nginx-error"
bootstrap_servers => "3.1.101.33:9092,3.1.101.34:9092,3.1.101.35:9092"
consumer_threads => 4
decorate_events => true
group_id => "kafka-nginx-error"
add_field => {"logstash-server" => "3.1.101.33"}
enable_metric => true
}
}
filter {
json {
source => "message"
}
grok {
match => [
"message", "%{DATESTAMP:currentDateTime}\s{1,}\[%{LOGLEVEL:level}\]\s{1,}(%{NUMBER:pid:int}#%{NUMBER}:\s{1,}\*%{NUMBER})\s{1,}(%{GREEDYDATA:messageInfo})(?:,\s{1,}client:\s{1,}(?<client>%{IP}|%{HOSTNAME}))(?:,\s{1,}server:\s{1,}%{IPORHOST:server})(?:, request: %{QS:request})?(?:, upstream: \"%{URI:endpoint}\")?(?:, host: \"%{HOSTPORT:host}\")?(?:, referrer: \"%{URI:referrer}\")?",
"message", "%{DATESTAMP:currentDateTime}\s{1,}\[%{DATA:level}\]\s{1,}%{GREEDYDATA:messageInfo}"]
}
date{
match => ["currentDateTime", "yy/MM/dd HH:mm:ss", "ISO8601"]
timezone => "+08:00"
target => "@timestamp"
}
mutate{
enable_metric => "false"
remove_field => [ "ecs","tags","input","agent","@version","log","port","host","message" ]
}
}
output {
elasticsearch {
hosts => ["3.1.101.33:9200","3.1.101.34:9200","3.1.101.35:9200"]
index => "logstash-nginx-error_%{+YYY-MM-dd}"
sniffing => true
template_overwrite => true
}
}
配置解析:
json插件: 将message中所有字段信息提取到顶级目录, 包括filebeat中定义的字段 grok插件: 做了一个match匹配列表, 来适应nginx错误日志的多种情况 date: match: 将提取的currentDateTime时间转换为ISO8601 timezone: 调整时区, 到kibana里会加8小时 target: 将currentDateTime时间覆盖@timestamp时间戳 mutate: enable_metric: 关闭指标 remove_field: 删除无用字段
过滤结果
JAVA
{
"_index": "index-ccms-dev_2022-05-31",
"_type": "_doc",
"_id": "MliXF4EBEBn_sZq0n-NG",
"_version": 1,
"_score": 1,
"_source": {
"class": "cn.git.collateral.manage.CollateralEsbApiImpl",
"kafka_topic": "topic-ccms-dev",
"logstash-server": "3.1.101.33",
"level": "ERROR",
"location": "CollateralEsbApiImpl.java,44,cn.git.collateral.manage.CollateralEsbApiImpl,esbOptionServer$original$rX5ytHc4",
"@timestamp": "2022-05-31T00:50:14.741Z",
"filebeat-server": "3.1.101.56",
"hostIp": "3.1.101.56",
"traceInfo": "TID: 97353afd28624015baa3f69c98ef23a7.93.16539582147090015",
"hostName": "ccms-test-18",
"currentDateTime": "2022-05-31T08:50:14.741+08:00",
"throwable": "''",
"messageInfo": "esb服务调用COLLATERAL模块异常,错误信息[nested exception is org.apache.ibatis.type.TypeException: Could not set parameters for mapping: ParameterMapping{property='colTpCd', mode=IN, javaType=class java.lang.Object, jdbcType=null, numericScale=null, resultMapId='null', jdbcTypeName='null', expression='null'}. Cause: org.apache.ibatis.type.TypeException: Error setting null for parameter #1 with JdbcType OTHER . Try setting a different JdbcType for this parameter or a different jdbcTypeForNull configuration property. Cause: java.sql.SQLException: 无效的列类型: 1111]",
"applicationName": "collateral-server"
}
}
NGINX
access
{
"_index": "logstash-nginx-access_2022-05-30",
"_type": "_doc",
"_id": "Q1RpEoEBEBn_sZq0ngYQ",
"_version": 1,
"_score": 1,
"_source": {
"file_dir": "/opt/ccms-auto-deploy/front-bank-credit/static/js/app.eef69865.js",
"log": {
"offset": 2433145,
"file": {
"path": "/data/openresty-vts/nginx/logs/ccms-credit-access.log"
}
},
"upstreamhost": "",
"request_body": "",
"logstash-server": "3.1.101.33",
"geoip": {
"postal_code": "98109",
"region_name": "Washington",
"location": {
"lat": 47.6348,
"lon": -122.3451
},
"city_name": "Seattle",
"coordinates": [
-122.3451
,
47.6348
],
"country_name": "United States",
"ip": "3.2.26.243"
},
"referer": "http://3.1.101.57:8901/",
"protocol": "HTTP/1.1",
"http_user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5005.62 Safari/537.36",
"request_length": "448",
"client_ip": "3.2.26.243",
"server_ip": "3.1.101.57",
"domain": "3.1.101.57",
"kafka_topic": "topic-nginx-access",
"url": "/static/js/app.eef69865.js",
"@timestamp": "2022-05-30T00:41:56.000Z",
"request_method": "GET",
"status": 304,
"responsetime": 0,
"xff": "",
"args": "",
"upstreamtime": 0,
"size": 0,
"ua": {
"name": "Chrome",
"os_name": "Windows",
"device": "Other",
"os": "Windows"
}
}
}
error
{
"_index": "logstash-nginx-error_2022-05-30",
"_type": "_doc",
"_id": "d_vBE4EBA-RkhKPLS3nJ",
"_version": 1,
"_score": 1,
"_source": {
"logstash-server": "3.1.101.34",
"request": ""GET /sysmanage/manage/organization/tree HTTP/1.1"",
"@timestamp": "2022-05-30T06:57:23.485Z",
"messageInfo": "an upstream response is buffered to a temporary file /usr/local/openresty/nginx/proxy_temp/3/14/0000000143 while reading upstream",
"kafka_topic": "topic-nginx-error",
"level": "warn",
"referrer": "http://3.1.101.57:8901/",
"client": "3.2.26.91",
"filebeat-server": "3.1.101.55",
"pid": 13,
"currentDateTime": "2022/05/28 09:57:46",
"endpoint": "http://3.1.101.56:11102/manage/organization/tree",
"server": "3.1.101.57"
}
}