Datax安装与使用及动态传参

直接下载DataX工具包：DataX下载地址

下载后解压至本地某个目录，进入bin目录，即可运行同步作业：

$ cd  {YOUR_DATAX_HOME}/bin
$ python datax.py {YOUR_JOB.json}

样例:TxtFile与TxtFile之间的传输

{
    "setting": {},
    "job": {
        "setting": {
            "speed": {
                "channel": 2
            }
        },
        "content": [
            {
                "reader": {
                    "name": "txtfilereader",
                    "parameter": {
                        "path": ["/data1_1T/shell/sync/data/tmp/sync_hive_to_mysql/ods_email_data"],
                        "encoding": "UTF-8",
                        "column": ["*"],
                        "fieldDelimiter": "\t"
                    }
                },
                "writer": {
                    "name": "txtfilewriter",
                    "parameter": {
                        "path": "/data1_1T/shell/sync/data/tmp/sync_hive_to_mysql/result",
                        "fileName": "luohw",
                        "writeMode": "truncate",
                    }
                }
            }
        ]
    }
}

样例:TxtFileToMysql之间的传输

{
    "job": {
        "setting": {
            "speed": {
                "channel": 2
            }
        },
        "content": [{
            "reader": {
                "name": "txtfilereader",
                "parameter": {
                    "path": ["/data1_1T/shell/sync/data/tmp/sync_hive_to_mysql/ods_email_data"],
                    "encoding": "UTF-8",
                    "column": ["*"],
                    "fieldDelimiter": "\t"
                }
            },
            "writer": {
                "name": "mysqlwriter",
                "parameter": {
                    "writeMode": "insert",
                    "username": "ark_code_read",
                    "password": "7opm7ii5pw",
                    "preSql": [
                        "delete from ods_email_data where ods_type='qfjx' "
                    ],
                    "column": ["ods_type", "val1", "val2", "device_count", "status", "created_date", "update_time"],
                    "connection": [{
                        "jdbcUrl": "jdbc:mysql://192.168.220.200:3306/analysys_meta?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true",
                        "table": ["ods_email_data"]
                    }]
                }
            }
        }]
    }
}

样例:MysqlToHdfs之间的传输
动态传参

$ cd  {YOUR_DATAX_HOME}/bin
$ python datax.py -p "-DTODAY=${TODAY}" {YOUR_JOB.json}

{
    "job": {
        "setting": {
            "speed": {
                 "channel":1
            }
        },
        "content": [
            {
                "reader": {
                    "name": "mysqlreader",
                    "parameter": {
                        "username": "xxxxx",
                        "password": "xxxxx",
                        "connection": [
                            {
                                "querySql": [
                                    "select * from dim_model_dict_ref where day=${TODAY};"
                                ],
                                "jdbcUrl": [
                                    "jdbc:mysql://000.000.220.200:3306/ecdc?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true"
                                ]
                            }
                        ]
                    }
                },
                "writer": {
                     "name": "hdfswriter",
                     "parameter": {
                        "defaultFS": "hdfs://zk14",
                        "fileType": "text",
                        "path":"/data/tmp/sync/dim_model_dict_ref",
                        "fileName": "dim_model_dict_ref",
                        "column":[
                            {
                                "name": "id",
                                "type": "int"
                            },
                            {
                                "name": "model",
                                "type": "string"
                            },
                            {
                                "name": "original_brand",
                                "type": "string"
                            },
                            {
                                "name": "stand_model_id",
                                "type": "string"
                            },
                            {
                                "name": "device_count",
                                "type": "int"
                            },
                            {
                                "name": "device_count_time",
                                "type": "string"
                            },
                            {
                                "name": "is_processed",
                                "type": "int"
                            },
                            {
                                "name": "sort",
                                "type": "int"
                            },
                            {
                                "name": "status",
                                "type": "int"
                            },
                            {
                                "name": "comment_info",
                                "type": "string"
                            },
                            {
                                "name": "created_date",
                                "type": "string"
                            },
                            {
                                "name": "update_time",
                                "type": "string"
                            },
                            {
                                "name": "created_by",
                                "type": "string"
                            },
                            {   "name": "updated_by",
                                "type": "string"
                            }
                        ],
                        "writeMode": "append",
                        "fieldDelimiter": "\t"
                      }
                }
            }
        ]
    }
}

详细参见官方文档