测试key-value

package com.datacloudsec.test.collect;

import com.alibaba.fastjson.JSON;
import com.datacloudsec.UEBAApplication;
import com.datacloudsec.collector.DcFacadeUtil;
import com.datacloudsec.collector.collector.source.parser.DefaultParser;
import com.datacloudsec.collector.collector.source.parser.KeyValueParser;
import com.datacloudsec.collector.collector.source.parser.Parser;
import com.datacloudsec.collector.common.event.Event;
import com.datacloudsec.collector.config.DcDecodeConfig;
import com.datacloudsec.collector.collector.repository.entity.Collector;
import com.datacloudsec.collector.collector.service.CollectorService;
import com.datacloudsec.event.repo.entity.EventDecodeRule;
import com.datacloudsec.event.service.EventDecodeRuleFieldMappingService;
import com.datacloudsec.event.service.EventDecodeRuleFieldService;
import com.datacloudsec.event.service.EventDecodeRuleService;
import com.datacloudsec.event.service.EventTypeService;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;

import java.util.Date;
import java.util.List;
import java.util.stream.Collectors;

/**
 * @Author xlj
 * @Date 2018/4/9 19:48
 */

@RunWith(SpringJUnit4ClassRunner.class)
@SpringBootTest(classes = UEBAApplication.class)
public class SpringKvTest {
    @Autowired
    private EventTypeService eventTypeService;

    @Autowired
    private EventDecodeRuleService eventDecodeRuleService;

    @Autowired
    private EventDecodeRuleFieldService eventDecodeRuleFieldService;

    @Autowired
    private EventDecodeRuleFieldMappingService eventDecodeRuleFieldMappingService;

    @Autowired
    private CollectorService collectorService;

    @Test
    public void defaultParseTest() {
        /**
         * 7
         * 安数云WAF解析规则_审计日志
         * 15
         * 4
         * <389>Oct  2 16:37:00 host CONFIG: SerialNum="0003211412129999001" GenTime="2017-10-02 16:37:00" SIP=192.168.101.70 DIP=192.168.101.89 UserName="adm" Operate="显示配置" ManageStyle=WEB Content="显示日志过滤配置" Log_Count="1"
         * 1
         * 10
         * <\d+>[a-zA-Z]{3}\s+\d+\s\d+:\d+:\d+\s+host\s+([a-z0-9A-Z_]+):\s+SerialNum="(.+)"\s+GenTime="(\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2})"\s+SIP=([0-9a-fA-F.:]+)\s+DIP=([0-9a-fA-F.:]+)\s+UserName="(.+)"\sOperate="(.+)"\s+ManageStyle=([a-zA-Z0-9]+)\s+Content="(.+)"\s+Log_Count="(\d+)"
         * 2017-09-20 17:11:02 2017-10-25 11:02:51
         */
        EventDecodeRule rule = new EventDecodeRule();
        rule.setId(7);
        rule.setRuleName("安数云WAF解析规则_审计日志");
        rule.setEventTypeId(15);
        rule.setAssetTypeId(4);
        rule.setOriginLog("<389>Oct  2 16:37:00 host CONFIG: SerialNum=\"0003211412129999001\" GenTime=\"2017-10-02 16:37:00\" SIP=192.168.101.70 DIP=192.168.101.89 UserName=\"adm\" Operate=\"显示配置\" ManageStyle=WEB Content=\"显示日志过滤配置\" Log_Count=\"1\"");
        rule.setDecodeType(1);
        rule.setAssetTypeId(10);
        rule.setRegex("<\\d+>[a-zA-Z]{3}\\s+\\d+\\s\\d+:\\d+:\\d+\\s+host\\s+([a-z0-9A-Z_]+):\\s+SerialNum=\"(.+)\"\\s+GenTime=\"(\\d{4}-\\d{2}-\\d{2}\\s+\\d{2}:\\d{2}:\\d{2})\"\\s+SIP=([0-9a-fA-F.:]+)\\s+DIP=([0-9a-fA-F.:]+)\\s+UserName=\"(.+)\"\\sOperate=\"(.+)\"\\s+ManageStyle=([a-zA-Z0-9]+)\\s+Content=\"(.+)\"\\s+Log_Count=\"(\\d+)\"");
        rule.setInsertTime(new Date());

        initDecodeConfig();
        Parser parser = new DefaultParser.Builder().build(rule);
        List<Event> event = parser.parse("127.0.0.1", rule.getOriginLog());
        System.out.println(JSON.toJSONString(event));
    }

    @Test
    public void kvParseTest() {
        /**
         * 33
         * 测试key-value审计日志
         * 15
         * 4
         * <190>May 18 11:20:10 2016 HLJ_S12508_1_FW %%10FILTER/6/ZONE_DP_FLT_EXECUTION_TCP_LOG(l): -DEV_TYPE=SECPATH-PN=210231A0H6010C000002;
         * srcZoneName=serveruntrust;
         * destZoneName(1035)=servertrust;rule_ID(1070)=90;
         * policyActType(1071)=denied;protType(1001)=TCP(6);
         * srcIPAddr(1017)=10.167.77.99;
         * destIPAddr(1019)=10.166.5.70;
         * srcPortNum(1018)=49362;
         * destPortNum(1020)=1521;
         * beginTime_e(1013)=05182016112009;
         * endTime_e(1014)=05182016112009;
         * Content=[HTTP_SQL_注入攻击(1&1)]red_begin URL::1=1%20or%202=2 red_end ;
         * HOST=web.chacuo.net;URL=/formatxml?1=1%20or%202=2;
         * REF=;
         * 2
         * 10
         * <[\S]+>(?<timestamp>\S+\s+\S+\s+\S+\s+\S+\s+)\S+ \%\%(?<vendor>[^/]*)/(?<severity>[^/]*)/(?<MNEMONIC>[^:]*): -DEV_TYPE=SECPATH-PN=210231A0H6010C000002; (?<message>.*)
         * 2017-09-20 17:11:02
         * 2017-10-25 11:02:51
         * 2
         * [^=;]+
         * =
         * [^;]+
         */
        EventDecodeRule rule = new EventDecodeRule();
        rule.setId(33);
        rule.setRuleName("安数云WAF解析规则_审计日志");
        rule.setEventTypeId(15);
        rule.setAssetTypeId(4);
        rule.setOriginLog("<190>May 18 11:20:10 2016 HLJ_S12508_1_FW %%10FILTER/6/ZONE_DP_FLT_EXECUTION_TCP_LOG(l): -DEV_TYPE=SECPATH-PN=210231A0H6010C000002; srcZoneName(1034)=serveruntrust;destZoneName(1035)=servertrust;rule_ID(1070)=90;policyActType(1071)=denied;protType(1001)=TCP(6);srcIPAddr(1017)=10.167.77.99;destIPAddr(1019)=10.166.5.70;srcPortNum(1018)=49362;destPortNum(1020)=1521;beginTime_e(1013)=05182016112009;endTime_e(1014)=05182016112009;Content=[HTTP_SQL_注入攻击(1&1)]red_begin URL::1=1%20or%202=2 red_end ;HOST=web.chacuo.net;URL=/formatxml?1=1%20or%202=2;REF=;" +
                "---<190>May 18 11:20:10 2016 HLJ_S12508_1_FW %%10FILTER/6/ZONE_DP_FLT_EXECUTION_TCP_LOG(l): -DEV_TYPE=SECPATH-PN=210231A0H6010C000002; srcZoneName(1034)=serveruntrust;destZoneName(1035)=servertrust;rule_ID(1070)=90;policyActType(1071)=denied;protType(1001)=TCP(6);srcIPAddr(1017)=10.167.77.99;destIPAddr(1019)=10.166.5.70;srcPortNum(1018)=49362;destPortNum(1020)=1521;beginTime_e(1013)=05182016112009;endTime_e(1014)=05182016112009;Content=[HTTP_SQL_注入攻击(1&1)]red_begin URL::1=1%20or%202=2 red_end ;HOST=web.chacuo.net;URL=/formatxml?1=1%20or%202=2;REF=;" +
                "---<190>May 18 11:20:10 2016 HLJ_S12508_1_FW %%10FILTER/6/ZONE_DP_FLT_EXECUTION_TCP_LOG(l): -DEV_TYPE=SECPATH-PN=210231A0H6010C000002; srcZoneName(1034)=serveruntrust;destZoneName(1035)=servertrust;rule_ID(1070)=90;policyActType(1071)=denied;protType(1001)=TCP(6);srcIPAddr(1017)=10.167.77.99;destIPAddr(1019)=10.166.5.70;srcPortNum(1018)=49362;destPortNum(1020)=1521;beginTime_e(1013)=05182016112009;endTime_e(1014)=05182016112009;Content=[HTTP_SQL_注入攻击(1&1)]red_begin URL::1=1%20or%202=2 red_end ;HOST=web.chacuo.net;URL=/formatxml?1=1%20or%202=2;REF=;");
        rule.setDecodeType(2);
        rule.setAssetTypeId(10);
        rule.setRegex("<[\\S]+>(?<timestamp>\\S+\\s+\\S+\\s+\\S+\\s+\\S+\\s+)\\S+ \\%\\%(?<vendor>[^/]*)/(?<severity>[^/]*)/(?<MNEMONIC>[^:]*): -DEV_TYPE=SECPATH-PN=210231A0H6010C000002; (?<message>.*)");
        rule.setInsertTime(new Date());
        rule.setSourceField("message");
        //多行设置，有则设置，无则不设置
        rule.setMultilineSeparator("---");

        //第一种 kv 分解
//        rule.setKvType(1);
//        rule.setKvSeparator("=");
//        rule.setFieldSeparator(";");
        //第二种 kv 正则解析
        rule.setKvType(2);
        rule.setKeyRegexp("([^=;]+)");
        rule.setValueRegexp("([^;]+)");
        rule.setSeparatorRegexp("=");


        initDecodeConfig();
        Parser parser = new KeyValueParser.Builder().build(rule);
        List<Event> event = parser.parse("127.0.0.1", rule.getOriginLog());
        System.out.println(JSON.toJSONString(event));
    }

    /**
     * 初始化解析配置
     */

    private void initDecodeConfig() {
        DcDecodeConfig.initEventTypes(eventTypeService.queryAll());
        DcDecodeConfig.initDecodeRules(eventDecodeRuleService.queryAll());
        DcDecodeConfig.initDecodeRuleFields(eventDecodeRuleFieldService.queryAll());
        DcDecodeConfig.initDecodeRuleFieldMappings(eventDecodeRuleFieldMappingService.queryAll());
        List<Collector> allCollectors = collectorService.queryAll()
                .stream().filter(c -> c.getEnable() == 1).collect(Collectors.toList());
        DcDecodeConfig.initCollectors(allCollectors);
        // 初始化所有解析器
        DcFacadeUtil.initAllParser();

    }
}