DataX的使用场景及配置文件

风中带血

已于 2024-04-07 11:02:20 修改

阅读量402

点赞数 1

文章标签： java kubernetes 大数据 mysql 开发语言

于 2024-04-07 10:59:07 首次发布

本文链接：https://blog.csdn.net/AKANGYONG/article/details/137455593

版权

查询

数据库同步到数据库 (自定义SQL)

{
    "job": {
        "setting": {
            "errorLimit":{
                "percentage":0,
                "record":0
            },
            "speed":{
                "channel":5
            }
        },
        "content":[
            {
                "reader":{
                    "name":"mysqlreader",
                    "parameter":{
                        "username": "username",
                        "password": "password",
                        "connection": [
                          {
                            "jdbcUrl": ["jdbc:mysql://ip:port/databse"],
                            "querySql": ["select value1,value2 from tableName where 1=1"]
                             }
                        ]
                    }
                },
                "writer": {
                  "name": "mysqlwriter",
                  "parameter": {
                    "column": [
                                "value1",
                                "value2"
                            ],
                    "encoding": "UTF-8",
                    "connection": [
                      {
                        "jdbcUrl": "jdbc:mysql://127.0.0.1:3306/test",
                        "table":["tableName"]
                         }
                    ],
                    "username": "username",
                    "password": "password",
                    "preSql":[
                        "前置SQL"
                    ],
                    "postSql":[
                        "后置SQL"
                    ]
                  }
                }
            }
        ]
    }
}

数据库同步到数据库 (固定字段)

{
    "core": {
         "transport" : {
              "channel": {
                   "speed": {
                       "record": 1000
                    }
               }
         }
    },
    "job": {
        "setting": {
            "errorLimit":{
                "percentage":0,
                "record":0
            },
            "speed":{
                "channel":2,
                "record":1000
            }
        },
        "content":[
            {
                "reader":{
                    "name":"mysqlreader",
                    "parameter":{
                        "username": "username",
                        "password": "password",
                        "connection": [
                          {
                            "jdbcUrl": ["jdbc:mysql://ip:port/database"],
                            "table": ["table1"]
                          }
                        ],
                        "column": ["*"],
                        "where": "1=1"
                    }
                },
                "writer": {
                  "name": "mysqlwriter",
                  "parameter": {
                    "column": ["*"],
                    "encoding": "UTF-8",
                    "connection": [
                      {
                        "jdbcUrl": "jdbc:mysql://ip:port/database",
                        "table":["table2"]
                      }
                    ],
                    "password": "password",
                    "preSql":[
                        "前置sql"
                    ],
                    "username": "username"
                  }
                }
            }
        ]
    }
}

数据库同步到hdfs

{
    "job": {
        "setting": {
            "speed":{
                "channel":5
            }
        },
        "content":[
            {
                "reader":{
                    "name":"mysqlreader",
                    "parameter":{
                        "username": "username",
                        "password": "password",
                        "column":[
                            "value1",
                            "REPLACE(REPLACE(value2, CHAR(10), ''), CHAR(13), '')"
                        ],
                        "where": "valida_status ='1'",
                        "splitFactor": 3,
                        "samplePercentage": 1,
                        "connection": [
                          {
                            "table": [
                              "table_name"
                            ],
                            "jdbcUrl": [
                              "jdbc:mysql://ip:port/database"
                            ]
                          }
                        ]
                    }
                },
                "writer": {
                  "name": "hdfswriter",
                  "parameter": {
                    "column": [
                      {
                        "name": "value1",
                        "type": "string"
                      },
                      {
                        "name": "value2",
                        "type": "string"
                      }
                    ],
                    "encoding": "UTF-8",
                    "defaultFS": "hdfs://hdfsunity",
                    "fieldDelimiter": "\u0001",
                    "fileName": "fileName",
                    "fileType": "text",
                    "path": "filePath",
                    "writeMode": "truncate",
                    "hadoopConfig":{
                        "dfs.client.failover.proxy.provider.hdfsunity":"org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider",
                        "dfs.ha.namenodes.hdfsunity":"r1,r2",
                        "dfs.namenode.rpc-address.hdfsunity.r1":"",
                        "dfs.namenode.rpc-address.hdfsunity.r2":"",
                        "dfs.nameservices":"hdfsunity",
                        "fs.defaultFS":"hdfs://hdfsunity"
                    },
                    "haveKerberos":true,
                    "kerberosKeytabFilePath":"",
                    "kerberosKrb5FilePath":"",
                    "kerberosPrincipal":"",
                  }
                }
            }
        ]
    }
}

hdfs同步到数据库

{
    "job": {
        "setting": {
            "speed":{
                "channel":5
            }
        },
        "content":[
            {
                "reader":{
                    "name":"hdfsreader",
                    "parameter":{
                        "column":[
                            {
                                "index":0,
                                "type":"string"
                            },
                            {
                                "index":1,
                                "type":"string"
                            }
                        ],
                        "defaultFS":"hdfs://hdfsunity",
                        "encoding":"UTF-8",
                        "fieldDelimiter":"\u0001",
                        "fileType":"TEXT",
                        "compress": "hadoop-snappy",
                        "hadoopConfig":{
                            "dfs.client.failover.proxy.provider.hdfsunity":"org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider",
                            "dfs.ha.namenodes.hdfsunity":"r1,r2",
                            "dfs.namenode.rpc-address.hdfsunity.r1":"",
                            "dfs.namenode.rpc-address.hdfsunity.r2":"",
                            "dfs.nameservices":"hdfsunity",
                            "fs.defaultFS":"hdfs://hdfsunity"
                        },
                        "haveKerberos":true,
                        "kerberosKeytabFilePath":"",
                        "kerberosKrb5FilePath":"",
                        "kerberosPrincipal":"",
                        "path":"filePath"
                    }
                },
                "writer":{
                    "name":"mysqlwriter",
                    "parameter":{
                        "column":[
                            "value1",
                            "value2"
                        ],
                        "connection":[
                            {
                                "jdbcUrl": "jdbc:mysql://ip:port/database",
                                "table": ["tableName"]
                            }
                        ],
                        "username":"USERNAME",
                        "password":"PASSWORD",
                        "preSql":[
                            "前置SQL"
                        ],
                        "postSql":[
                            "后置SQL"
                        ]
                    }
                }
            }
        ]
    }
}

update

{
    "job": {
        "setting": {
            "speed": {
                "channel": 5
            }
        },
        "content": [
            {
                "reader": {
                    "name": "mysqlreader",
                    "parameter": {
                        "username": "username",
                        "password": "password",
                        "connection": [
                            {
                                "querySql": [
                                    "UPDATE ti_tower_modeling_in_init_pro SET status_code = 9 WHERE unicom_fee IS NULL"
                                ],
                                "jdbcUrl": [
                                    "jdbc:mysql://ip:port/database"
                                ]
                            }
                        ]
                    }
                },
                "writer": {
                    "name": "streamwriter",
                    "parameter": {
                        "print": true
                    }
                }
            }
        ]
    }
}