logstash 解析 xml- 示例

最新推荐文章于 2020-10-12 15:39:45 发布

weixin_34255793

最新推荐文章于 2020-10-12 15:39:45 发布

阅读量1.8k

点赞数

文章标签： ruby python 大数据

原文链接：https://my.oschina.net/ch66880/blog/1617390

版权

2019独角兽企业重金招聘Python工程师标准>>>

源文件:

<?xml version="1.0" encoding="UTF-8"?>
<IPC2547Event><ProcessSessionStart dateTime="2017-09-07T11:37:23.00+08:00" sessionId="S_20200008618100004_1504755459">
<Product itemType="852RN5691MB2003"/>
<Entity stationId="TBA7150495" stage="AOI"/>
<Recipe recipeId="852RN5691MB2003.tst" revision="2017-09-07T09:42:35.00+08:00">
<RecipeModule moduleId="456.bib" revision="2017-09-07T10:48:48.00+08:00"</Recipe></ProcessSessionStart>
<ItemProcessStatus dateTime="2017-09-07T11:37:23.00+08:00" itemInstanceId="20200008618100004" sessionRef="S_20200008618100004_1504755459" itemProcessId="20200008618100004_1504755459" status="PASSED">
<ItemEventCount eventType="PROCESSSTEPSTATUS" count="2"/>
</ItemProcessStatus>
<ProcessStepStatus dateTime="2017-09-07T11:37:23.00+08:00" itemInstanceId="20200008618100004" sessionRef="S_20200008618100004_1504755459" itemProcessRef="20200008618100004_1504755459" processStepId="20200008618100004_1" status="PASSED" imageId="1"/>
<ProcessStepStatus dateTime="2017-09-07T11:37:23.00+08:00" itemInstanceId="20200008618100004" sessionRef="S_20200008618100004_1504755459" itemProcessRef="20200008618100004_1504755459" processStepId="20200008618100003" status="PASSED" imageId="2"/>
</IPC2547Event>

解析配置如下：


input {
  stdin{ }
}
#解析文件
filter {
  xml {
remove_namespaces=>"true"
        source => "message"
        target =>"doc"
        xpath => [
                "/IPC2547Event/ItemProcessStatus/@dateTime","dateTime",
"/IPC2547Event/ItemProcessStatus/@itemInstanceId","itemInstanceId",
"/IPC2547Event/ItemProcessStatus/@sessionRef","sessionRef",
"/IPC2547Event/ItemProcessStatus/@itemProcessId","itemProcessId",
"/IPC2547Event/ItemProcessStatus/@status","status",


"/IPC2547Event/ItemProcessStatus/ItemEventCount/@eventType","eventType",
"/IPC2547Event/ItemProcessStatus/ItemEventCount/@count","count",

"/IPC2547Event/ProcessStepStatus[1]/@dateTime","dateTime",
"/IPC2547Event/ProcessStepStatus[1]/@itemInstanceId","itemInstanceId",
"/IPC2547Event/ProcessStepStatus[1]/@sessionRef","sessionRef",
"/IPC2547Event/ProcessStepStatus[1]/@itemProcessRef","itemProcessRef",
"/IPC2547Event/ProcessStepStatus[1]/@processStepId","processStepId",
"/IPC2547Event/ProcessStepStatus[1]/@status","status",
"/IPC2547Event/ProcessStepStatus[1]/@imageId","imageId",

"/IPC2547Event/ProcessStepStatus[2]/@dateTime","dateTime",
"/IPC2547Event/ProcessStepStatus[2]/@itemInstanceId","itemInstanceId",
"/IPC2547Event/ProcessStepStatus[2]/@sessionRef","sessionRef",
"/IPC2547Event/ProcessStepStatus[2]/@itemProcessRef","itemProcessRef",
"/IPC2547Event/ProcessStepStatus[2]/@processStepId","processStepId",
"/IPC2547Event/ProcessStepStatus[2]/@status","status",
"/IPC2547Event/ProcessStepStatus[2]/@imageId","imageId"
                ]
        }
  }
output {
    stdout{
        codec=>rubydebug
 }
}

往ES中写入数据:

filter {
  xml {
remove_namespaces=>"true"
        source => "message"
        target =>"doc"
        xpath => [
                "/IPC2547Event/ItemProcessStatus/@dateTime","dateTime",
"/IPC2547Event/ItemProcessStatus/@itemInstanceId","itemInstanceId",
"/IPC2547Event/ItemProcessStatus/@sessionRef","sessionRef",
"/IPC2547Event/ItemProcessStatus/@itemProcessId","itemProcessId",
"/IPC2547Event/ItemProcessStatus/@status","status",


"/IPC2547Event/ItemProcessStatus/ItemEventCount/@eventType","eventType",
"/IPC2547Event/ItemProcessStatus/ItemEventCount/@count","count",

"/IPC2547Event/ProcessStepStatus[1]/@dateTime","dateTime",
"/IPC2547Event/ProcessStepStatus[1]/@itemInstanceId","itemInstanceId",
"/IPC2547Event/ProcessStepStatus[1]/@sessionRef","sessionRef",
"/IPC2547Event/ProcessStepStatus[1]/@itemProcessRef","itemProcessRef",
"/IPC2547Event/ProcessStepStatus[1]/@processStepId","processStepId",
"/IPC2547Event/ProcessStepStatus[1]/@status","status",
"/IPC2547Event/ProcessStepStatus[1]/@imageId","imageId",

"/IPC2547Event/ProcessStepStatus[2]/@dateTime","dateTime",
"/IPC2547Event/ProcessStepStatus[2]/@itemInstanceId","itemInstanceId",
"/IPC2547Event/ProcessStepStatus[2]/@sessionRef","sessionRef",
"/IPC2547Event/ProcessStepStatus[2]/@itemProcessRef","itemProcessRef",
"/IPC2547Event/ProcessStepStatus[2]/@processStepId","processStepId",
"/IPC2547Event/ProcessStepStatus[2]/@status","status",
"/IPC2547Event/ProcessStepStatus[2]/@imageId","imageId"
                ]
        }
  }
output {
        elasticsearch {
    index => "log-%{+YYYY.MM.dd}"
    hosts => ["10.2.1.238:9200", "10.2.1.237:9200", "10.2.1.241:9200"]
    }
    stdout{
        codec=>rubydebug
        }
}

解析结果：

{
      "@version" => "1",
          "host" => "localhost.localdomain",
    "@timestamp" => 2018-02-02T06:38:24.163Z,
       "message" => "<?xml version=\"1.0\" encoding=\"UTF-8\"?>",
          "tags" => [
        [0] "_xmlparsefailure"
    ]
}
{
          "dateTime" => [
        [0] "2017-09-07T11:37:23.00+08:00",
        [1] "2017-09-07T11:37:23.00+08:00",
        [2] "2017-09-07T11:37:23.00+08:00"
    ],
    "itemInstanceId" => [
        [0] "20200008618100004",
        [1] "20200008618100004",
        [2] "20200008618100004"
    ],
           "imageId" => [
        [0] "1",
        [1] "2"
    ],
             "count" => [
        [0] "2"
    ],
         "eventType" => [
        [0] "PROCESSSTEPSTATUS"
    ],
           "message" => "<IPC2547Event><ProcessSessionStart dateTime=\"2017-09-07T11:37:23.00+08:00\" sessionId=\"S_20200008618100004_1504755459\"><Product itemType=\"852RN5691MB2003\"/><Entity stationId=\"TBA7150495\" stage=\"AOI\"/><Recipe recipeId=\"852RN5691MB2003.tst\" revision=\"2017-09-07T09:42:35.00+08:00\"><RecipeModule moduleId=\"456.bib\" revision=\"2017-09-07T10:48:48.00+08:00\"/></Recipe></ProcessSessionStart><ItemProcessStatus dateTime=\"2017-09-07T11:37:23.00+08:00\" itemInstanceId=\"20200008618100004\" sessionRef=\"S_20200008618100004_1504755459\" itemProcessId=\"20200008618100004_1504755459\" status=\"PASSED\"><ItemEventCount eventType=\"PROCESSSTEPSTATUS\" count=\"2\"/></ItemProcessStatus><ProcessStepStatus dateTime=\"2017-09-07T11:37:23.00+08:00\" itemInstanceId=\"20200008618100004\" sessionRef=\"S_20200008618100004_1504755459\" itemProcessRef=\"20200008618100004_1504755459\" processStepId=\"20200008618100004_1\" status=\"PASSED\" imageId=\"1\"/><ProcessStepStatus dateTime=\"2017-09-07T11:37:23.00+08:00\" itemInstanceId=\"20200008618100004\" sessionRef=\"S_20200008618100004_1504755459\" itemProcessRef=\"20200008618100004_1504755459\" processStepId=\"20200008618100003\" status=\"PASSED\" imageId=\"2\"/></IPC2547Event>",
        "sessionRef" => [
        [0] "S_20200008618100004_1504755459",
        [1] "S_20200008618100004_1504755459",
        [2] "S_20200008618100004_1504755459"
    ],
     "processStepId" => [
        [0] "20200008618100004_1",
        [1] "20200008618100003"
    ],
        "@timestamp" => 2018-02-02T06:38:24.166Z,
          "@version" => "1",
              "host" => "localhost.localdomain",
               "doc" => {
          "ItemProcessStatus" => [
            [0] {
                      "dateTime" => "2017-09-07T11:37:23.00+08:00",
                    "sessionRef" => "S_20200008618100004_1504755459",
                "itemInstanceId" => "20200008618100004",
                 "itemProcessId" => "20200008618100004_1504755459",
                "ItemEventCount" => [
                    [0] {
                            "count" => "2",
                        "eventType" => "PROCESSSTEPSTATUS"
                    }
                ],
                        "status" => "PASSED"
            }
        ],
        "ProcessSessionStart" => [
            [0] {
                 "dateTime" => "2017-09-07T11:37:23.00+08:00",
                  "Product" => [
                    [0] {
                        "itemType" => "852RN5691MB2003"
                    }
                ],
                   "Entity" => [
                    [0] {
                            "stage" => "AOI",
                        "stationId" => "TBA7150495"
                    }
                ],
                "sessionId" => "S_20200008618100004_1504755459",
                   "Recipe" => [
                    [0] {
                        "RecipeModule" => [
                            [0] {
                                "moduleId" => "456.bib",
                                "revision" => "2017-09-07T10:48:48.00+08:00"
                            }
                        ],
                            "recipeId" => "852RN5691MB2003.tst",
                            "revision" => "2017-09-07T09:42:35.00+08:00"
                    }
                ]
            }
        ],
          "ProcessStepStatus" => [
            [0] {
                      "dateTime" => "2017-09-07T11:37:23.00+08:00",
                    "sessionRef" => "S_20200008618100004_1504755459",
                 "processStepId" => "20200008618100004_1",
                "itemInstanceId" => "20200008618100004",
                       "imageId" => "1",
                "itemProcessRef" => "20200008618100004_1504755459",
                        "status" => "PASSED"
            },
            [1] {
                      "dateTime" => "2017-09-07T11:37:23.00+08:00",
                    "sessionRef" => "S_20200008618100004_1504755459",
                 "processStepId" => "20200008618100003",
                "itemInstanceId" => "20200008618100004",
                       "imageId" => "2",
                "itemProcessRef" => "20200008618100004_1504755459",
                        "status" => "PASSED"
            }
        ]
    },
     "itemProcessId" => [
        [0] "20200008618100004_1504755459"
    ],
            "status" => [
        [0] "PASSED",
        [1] "PASSED",
        [2] "PASSED"
    ],
    "itemProcessRef" => [
        [0] "20200008618100004_1504755459",
        [1] "20200008618100004_1504755459"
    ]
}

转载于:https://my.oschina.net/ch66880/blog/1617390