【Graylog】比较常用的pipeline规则

毫秒转换为yyyy-MM-dd HH:mm:ss日志格式

rule "receiveDate_alignment"
when
  has_field("receiveDate")
then
    let m = parse_unix_milliseconds(to_long($message.receiveDate),"CST");
    let n = format_date(m,"yyyy-MM-dd HH:mm:ss","Asia/Shanghai");
    set_field("ORG_EVENT_TIME",n);
end

时间格式化及时区格式化

rule "parse_log"
when
  contains(value: to_string($message.message), search: "ERROR", ignore_case: false)
then
  set_field("raw_message", to_string($message.message));
  set_field("message", concat("警告：匹配到深证通程序日志有异常关键字，请注意查看！",to_string(format_date(to_date($message.timestamp,"CST"),"yyyy-MM-dd HH:mm:ss","Asia/Shanghai"))));
end

format_date(to_date($message.timestamp,"CST"),"yyyy-MM-dd HH:mm:ss","Asia/Shanghai")

2022-03-03T06:46:26.354Z  => 2022-03-03 14:46:26

特殊时间格式对其（filebeat指标）

rule "time_duiqi2"
when
  true
then
  let m = regex("(\\S+)\\+",to_string($message.message));
  set_field("timestamp",parse_date(replace(to_string(m["0"]),"T"," "),"yyyy-MM-dd HH:mm:ss.SSS","locale.US","Asia/Shanghai"));
end

filebeat日志时间格式为2022-09-16T13:56:39.278+0800，由于中间带T操作时解析总无法实现，把T替换掉为空格即可

判断条件的跃进

rule "test rule"
when
  //可以进行数值判断
  to_long($message.alert_level) > 3
then
  set_field("test_field", "test succ");
end

判断条件的正则匹配（常用的contains不支持正则）

rule "src_ip_v6"
when
has_field("src_ip") &&
  regex("\\d+\\.\\d+\\.\\d+\\.\\d+", to_string($message.src_ip)).matches == false
then
    set_field("reason","maybe v6");
end

根据Pri分析出Facility和Level字段值

//把22解析成pri

<22>Jul 13 17:25:01 localhost postfix/qmgr[1915]: CB3FF60AF6F5: removed 

rule "conv"
when
    true
then
    let m = expand_syslog_priority_as_string(to_string($message.pri));
    set_fields({facility: m.facility, level:m.level});
end

查询表（lookuptable）功能使用csv适配器实现多值

csv文件格式如下

"http_response_code"|"multivalue"
"200"|"中国#测试"

规则如下：

rule "parse_lookuptable_multivalue"
when
  has_field("http_response_code")
then
  let lookup_multivalue = lookup("csv_test",to_string($message.http_response_code));
  let multivalue = split("#",to_string(lookup_multivalue.value));
  set_field("localte",to_string(multivalue[0]));
  set_field("desc",to_string(multivalue[1]));
end

效果如下：

JSON格式解析

rule "parse_json"
when
  true
then
  let m = parse_json(to_string($message.message));
  set_fields(to_map(m));
end

json格式解析（大json解析指定数据）

rule "parse"
when
  true
then
  let m = regex(".*?(\\{.*)",to_string($message.message));
  let x = parse_json(to_string(m["0"]));
  let new_fields = select_jsonpath(x,
            {
            load1: "$.monitoring.metrics.system.load.1",
            load5: "$.monitoring.metrics.system.load.5",
            load15: "$.monitoring.metrics.system.load.15"
            });
  set_fields(new_fields);
  
end

可以从以下json中获取load相关指标

2022-09-15T11:06:53.451+0800	INFO	[monitoring]	log/log.go:145	Non-zero metrics in the last 30s	{"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":1450,"time":{"ms":3}},"total":{"ticks":3160,"time":{"ms":25},"value":3160},"user":{"ticks":1710,"time":{"ms":22}}},"handles":{"limit":{"hard":65536,"soft":65536},"open":14},"info":{"ephemeral_id":"0aed24ea-c8cd-429a-ae16-b701558bebdb","uptime":{"ms":3870086}},"memstats":{"gc_next":23807808,"memory_alloc":12150784,"memory_total":160990448},"runtime":{"goroutines":43}},"filebeat":{"events":{"added":1,"done":1},"harvester":{"files":{"041fdc0c-82ad-495a-9244-54966725bfe5":{"last_event_published_time":"2022-09-15T11:06:29.727Z","last_event_timestamp":"2022-09-15T11:06:29.727Z","read_offset":1231,"size":1231}},"open_files":1,"running":1}},"libbeat":{"config":{"module":{"running":0}},"output":{"events":{"acked":1,"batches":1,"total":1}},"outputs":{"kafka":{"bytes_read":60,"bytes_write":1124}},"pipeline":{"clients":2,"events":{"active":0,"published":1,"total":1},"queue":{"acked":1}}},"registrar":{"states":{"current":2,"update":1},"writes":{"success":1,"total":1}},"system":{"load":{"1":0.09,"15":0.29,"5":0.27,"norm":{"1":0.0113,"15":0.0363,"5":0.0338}}}}}}

嵌套json解析

json示例

{"host":{"host":"192.168.100.12","name":"B-BJ-HW-S5720-03_04"},"groups":["Templates/Network devices","DT_network","网络设备","路由器"],"applications":["Interface GigabitEthernet1/0/2()"],"itemid":67697,"name":"Interface GigabitEthernet1/0/2(): Bits 接收","clock":1672724928,"ns":434058026,"value":46440,"type":3}

解析规则

rule "Data Parsing"
when
    true
then
    let msg = parse_json(to_string($message.message));
    let prop = select_jsonpath(msg, {host: "$.host"});
    set_field("host_string", to_string(prop.host));

    let props = parse_json(to_string($message.host_string));
    set_fields(to_map(props),"host_");
    set_fields(to_map(msg));
end

思路: 由于graylog4删减了部分函数，比如好用的nesting_parse_json()用于解析嵌套函数，所以通过社区找到一个案例照着改了一下。

大概的实现过程是这样的，首先先把所有的json外层解析出来，然后针对内层的host字段进行二次解析，解析后的内层host为了防止字段名称冲突导致覆盖或字段类型不同，在set_fields时加上前缀。