多种格式转json

# -*- coding: utf-8 -*-
import json
from lib import xmltodict


class LogToDict(object):

    @staticmethod
    def iter_data(data):
        """通用迭代器,支持dict和list"""
        if isinstance(data, dict):
            for key, value in data.items():
                yield (key, value)
        elif isinstance(data, list):
            for index, value in enumerate(data):
                yield (index, value)
        else:
            raise Exception("not support this type:{}".format(type(data)))

    @staticmethod
    def flat(data):
        """展平dict多层嵌套"""
        for key, value in LogToDict.iter_data(data):
            if isinstance(value, (dict, list)):
                for k, v in LogToDict.flat(value):
                    yield (k, v)
            else:
                yield (key, value)

    def xml_to_json(self, log):
        """原始日志为xml格式"""
        log = ' <xml> ' + log + '</xml>'
        content = json.loads(json.dumps(xmltodict.parse(log, encoding='utf-8', attr_prefix='')))['xml']
        data = {k: v for k, v in self.flat(content)}

        return data

    def list_to_json(self, log):
        """原始日志格式:[{},{}]"""

        content = json.loads(log)
        data = {k: v for k, v in self.flat(content)}
        return data

    def parse_log(self, log):
        """将原始日志解析为json格式"""
        try:
            data = self.list_to_json(log)
            return data
        except Exception as e:
            pass

        try:
            data = self.xml_to_json(log)
        except Exception as e:
            raise Exception("Failed to convert the original log to json format, {}".format(e.message))

        return data


if __name__ == '__main__':
    xml_log = '<headinfo product="nf" product_ip="10.14.11.36" hash="D762-ACD0-A25C-E748" msgtype="1"/><nidslog action="1" acted="1" sip="172.16.6.6" sport="49205" dip="172.18.6.6" dport="80" smac="00:10:F3:27:57:7E" dmac="00:0C:29:E6:8C:00" group="233898077" module="2" ruleid="50243" vid="0" lasttimes="1" date="1596695679" msel="0" rawlen="496" rawinfo="R0VUIC8gSFRUUC8xLjENCkhvc3Q6IDE3Mi4xOC42LjYNClVzZXItQWdlbnQ6IE1vemlsbGEvNS4wIChXaW5kb3dzIE5UIDYuMTsgV09XNjQ7IHJ2OjQyLjApIEdlY2tvLzIwMTAwMTAxIEZpcmVmb3gvNDIuMA0KQWNjZXB0OiB0ZXh0L2h0bWwsYXBwbGljYXRpb24veGh0bWwreG1sLGFwcGxpY2F0aW9uL3htbDtxPTAuOSwqLyo7cT0wLjgNCkFjY2VwdC1MYW5ndWFnZTogemgtQ04semg7cT0wLjgsZW4tVVM7cT0wLjUsZW47cT0wLjMNCkFjY2VwdC1FbmNvZGluZzogZ3ppcCwgZGVmbGF0ZQ0KUmVmZXJlcjogaHR0cDovLzE3Mi4xOC42LjYvDQpDb29raWU6IEhGU19TSURfPTAuMzk3MDU2NDQ3ODA3Njk5DQpDb25uZWN0aW9uOiBrZWVwLWFsaXZlDQoNCg==" msg="用户访问受控URL" ar="7995514" ds="VENQLkhUVFBuU2YwQ3VzQ25TZjBDdXPmnKrnn6VuU2YwQ3VzMTcyLjE4LjYuNm5TZjBDdXMv" card="G2/1" user="" />'

    list_log1 = '[{"hash": "719B-C06D-C37E-6E55", "product": "ids", "msgtype": 31489, "dev_ip": "10.67.1.131"}, {"msel": 0, "vid": 0, "acted": 65537, "module": 6, "smac": "00:12:3F:D4:C7:8B", "ar": 2, "rawinfo": "R0VUIC8gSFRUUC8xLjENCkFjY2VwdDogKi8qDQpBY2NlcHQtTGFuZ3VhZ2U6IHpoLUNODQpVc2VyLUFnZW50OiBNb3ppbGxhLzQuMCAoY29tcGF0aWJsZTsgTVNJRSA4LjA7IFdpbmRvd3MgTlQgNi4xOyBXT1c2NDsgVHJpZGVudC80LjA7IFNMQ0MyOyAuTkVUIENMUiAyLjAuNTA3Mjc7IC5ORVQgQ0xSIDMuNS4zMDcyOTsgLk5FVCBDTFIgMy4wLjMwNzI5OyBNZWRpYSBDZW50ZXIgUEMgNi4wKQ0KQWNjZXB0LUVuY29kaW5nOiBnemlwLCBkZWZsYXRlDQpIb3N0OiB3d3cucXEuY29tDQpDb25uZWN0aW9uOiBLZWVwLUFsaXZlDQpDb29raWU6IHRzX3VpZD0yNTIxMzEyMzQwOyBhZF9wbGF5X2luZGV4PTY7IF9hc19jcmF6eTQ0MDM9MjAxNC04LTE4LTItOyB0c19sYXN0PXd3dy5xcS5jb20vOyBwZ3ZfcHZpZD00MjQwMzQ4MjA1OyBhYz0xLDAxMiwwMTI7IHBndl9pbmZvPXNzaWQ9czU1OTEyMzExNjgNCg0K", "sport": 3298, "sip": "1.1.1.217", "group": 1, "ruleid": 41044, "msg": "5YO15bC4572R57ucK0lfQ184", "dmac": "00:15:FA:B4:E4:4C", "lasttimes": 9, "action": 1, "user": "", "date": 1468560536, "ds": "dGNwblNmMEN1c3d3dy5qZC5jb21uU2YwQ3VzMTExLjIwNi4yMzEuMW5TZjBDdXNhdHRhY2tfamQuY3Bw", "card": "G1/1", "rawlen": 684, "dport": 80, "dip": "2.2.2.171"}]'
    list_log2 = '{"srcOccurrence": {"12.131.8.141": 1}, "collector": ["F026-7A43-9027-AABC"], "id": "0fb36f2e6acd44f482cd0c1455047e8e", "devIp": "10.65.189.102", "sip": ["12.131.8.141"], "ruleid": "80e  b1d78aea943f9be5ff4066791b569", "dstOccurrence": {"10.65.189.102": 1}, "attackMeans": "软件缺陷", "valid": true, "fromid": ["a834953f-d9ed-42c7-aef0-5eec333ae597:test1711"], "category1": "网络攻击", "category2": "网络其他攻击", "risk": 1, "timestamp": 1598874176, "nsaction": 2, "alarmContent": "", "alarmNotice": false, "endtime": 1597294556, "alarmType": "威胁告警",   "stage": 1, "name": "自定义单源规则", "extension": [{"a":"null"}], "alarmEnable": true, "starttime": 1597294556, "alarmSuggestion": "", "attackMethod": "恶意样本", "dip": ["10.65.189.102"], "alarm_state": -1, "tagid": [null], "desc": "WebLogic是一个基于JAVAEE架构的中间件，webLogic是用于开发、集成、部署和管理大型分布式Web应用、网络应用和数据库应用的Java应用服务器。常用端口7001/7002，常见漏洞有弱口令，SSRF漏洞和反序列化漏洞，远程代码执行漏洞，任意文件上传漏洞等", "suggestion": "攻击者在尝试对weblogic漏洞做攻击尝试，如果被攻击的我方资产部署有weblogic，请持续关注该资产是否有后续的攻击事件产生。", "sport": [-1], "dport": [80], "src_country": ["中国"], "src_province": ["陕西"], "src_city": ["西安"], "dst_country": ["中国"], "dst_province": ["陕西"], "dst_city": ["西安"], "app_id": "event-stream", "soc_id": "test", "sasset": ["null"], "dasset": ["null"], "sassetgr": ["null"], "dassetgr": ["null"], "nstimestamp": [1597294803], "dst_organization": [0], "src_topology": [0], "dst_topology": [0], "src_business": [0], "dst_business": [0], "infohit": [{"a":"null"}], "focus": "null", "focusContent": ["null"], "fall_host": ["null"], "ntaEventId": 2147483, "sumPackets": 21474836, "nta_src_sum_packets": 922337, "alarm_no": 0, "sumBytes": 2147482, "ntaMaxBytes": 2147481, "alarm_rule_id": 1, "ntaEventType": 1, "ntaSeverity": 1, "domain": ["null"], "ntaMaxPackets": 2147480, "alarm_id": ["null"], "times_for_trap": 1, "nta_src_sum_bytes": 8547758, "invalid_for_trap": ["null"], "project_id": "1","nsaction": 1}'
    list_log3 = '{"data": [{"attack_ip": "112.168.1.1", "rx_packets": 124500, "stat_time": "2020-04-22 17:09:00", "rx_bytes": 6490500}], "tb_id": "5", "msgtype": "1111", "nta_ip": "10.67.1.236"}'

    log4 = '<14>[CEF]:|数据库名称:BJ_msp-elc|规则类型:SQL注入规则|风险级别:低风险|命中规则:SQL语句中包含注入常用的字符串函数|审计动作:审计|捕获时间:{{timestamp}}|服务器IP:110.0.5.5|服务器端口:1521|服务名（实例名）:gxb|客户端IP:110.0.3.214|客户端端口:63140|客户端MAC地址:A4FA76D681D2|数据库用户名:SF|操作系统用户名:root|SQL语句:select * from rate_limiter_info |执行次数:6'

    logs = [xml_log, list_log1, list_log2, list_log3, log4]
    logtodict = LogToDict()

    for log in logs:
        data = logtodict.parse_log(log)
        print(json.dumps(data))