DataWorks离线同步SLS数据到MaxCompute

最新推荐文章于 2024-10-29 09:57:15 发布

得得de

最新推荐文章于 2024-10-29 09:57:15 发布

阅读量191

点赞数

分类专栏：大数据文章标签：大数据 etl

本文链接：https://blog.csdn.net/feihangyu/article/details/133672572

版权

大数据专栏收录该内容

2 篇文章 0 订阅

订阅专栏

1、大体步骤阿里云官方文档已经很详细了

2、遇到一个问题

SLS没有自定义的时间字段，但是需要将日志时间作为一个字段同步到MC

{
    "type": "job",
    "version": "2.0",
    "steps": [
        {
            "stepType": "loghub",
            "parameter": {
                "datasource": "saas_internal_all_log",
                "envType": 1,
                "column": [
                    "bussinessType",
                    "data",
                    "identifier",
                    "imei",
                    "msgType",
                    "__time__"替换为C_LogTime
                ],
                "beginDateTime": "${bizdate}235900",
                "batchSize": 256,
                "endDateTime": "${bizdate}235959",
                "fieldDelimiter": ",",
                "encoding": "UTF-8",
                "logstore": "q-device-ebike-device-worker"
            },
            "name": "Reader",
            "category": "reader"
        },
        {
            "stepType": "odps",
            "parameter": {
                "partition": "pt=${bizdate}",
                "truncate": false,
                "datasource": "odps_first",
                "envType": 1,
                "isSupportThreeModel": false,
                "tunnelQuota": "default",
                "column": [
                    "bussinesstype",
                    "data",
                    "identifier",
                    "imei",
                    "msgtype",
                    "stime"
                ],
                "emptyAsNull": false,
                "tableComment": "null",
                "table": "xa_ods_sls_bin2_pt",
                "consistencyCommit": false
            },
            "name": "Writer",
            "category": "writer"
        },
        {
            "copies": 1,
            "parameter": {},
            "name": "Processor",
            "category": "processor"
        }
    ],
    "setting": {
        "executeMode": null,
        "errorLimit": {
            "record": ""
        },
        "speed": {
            "concurrent": 2,
            "throttle": false
        }
    },
    "order": {
        "hops": [
            {
                "from": "Reader",
                "to": "Writer"
            }
        ]
    }
}

同步字段为空，后将__time__替换为C_LogTime后成功