dataX json配置

最新推荐文章于 2024-10-12 09:03:02 发布

weixin_33980459

最新推荐文章于 2024-10-12 09:03:02 发布

阅读量2.4k

点赞数 1

文章标签： json 数据库大数据

原文链接：http://www.cnblogs.com/longbing/p/9430084.html

版权

1.mysql2mysql的例子

{
    "job":{
        "content":[
            {
                "reader":{
                    "name":"mysqlreader",
                    "parameter":{
                        "column":[
                            "id",
                            "name"
                        ],
                        "connection":[
                            {
                                "jdbcUrl":[
                                    "jdbc:mysql://127.0.0.1:3306/dq"
                                ],
                                "table":[
                                    "table1"
                                ]
                            }
                        ],
                        "password":"123456",
                        "username":"root"
                    }
                },
                "writer":{
                    "name":"mysqlwriter",
                    "parameter":{
                        "column":[
                            "id",
                            "name"
                        ],
                        "connection":[
                            {
                                "jdbcUrl":"jdbc:mysql://ip地址:端口/test",
                                "table":[
                                    "table2"
                                ]
                            }
                        ],
                        "password":"123456",
                        "username":"root"
                    }
                }
            }
        ],
        "setting":{
            "speed":{
                "channel":"1"
            }
        }
    }
}

2.mysql到本地：

{
    "job":{
        "content":[
            {
                "reader":{
                    "name":"mysqlreader",
                    "parameter":{
                        "username":"root",
                        "password":"root",
                        "column":[
                            "ID",
                            "VISIT_IP"
                        ],
                        "connection":[
                            {
                                "table":[
                                    "visit_log"
                                ],
                                "jdbcUrl":[
                                    "jdbc:mysql://192.168.0.197:3306/qyyx_test"
                                ]
                            }
                        ]
                    }
                },
                "writer":{
                    "name":"streamwriter",
                    "parameter":{
                        "print":false,
                        "encoding":"UTF-8"
                    }
                }
            }
        ]
    }
}

3.HDFS打印到本地控制台

{
    "job":{
        "setting":{
            "speed":{
                "channel":3
            }
        },
        "content":[
            {
                "reader":{
                    "name":"hdfsreader",
                    "parameter":{
                        "path":"/user/hive/warehouse/test.db/nnn/*",
                        "defaultFS":"hdfs://192.168.200.45:8020",
                        "column":[
                            "*"
                        ],
                        "fileType":"orc",
                        "encoding":"UTF-8",
                        "fieldDelimiter":","
                    }
                },
                "writer":{
                    "name":"streamwriter",
                    "parameter":{
                        "print":true
                    }
                }
            }
        ]
    }
}

4.orcle打印到Linux

{
    "job":{
        "setting":{
            "speed":{
                "channel":5
            }
        },
        "content":[
            {
                "reader":{
                    "name":"oraclereader",
                    "parameter":{
                        "username":"edw_src",
                        "password":"123456",
                        "where":"",
                        "connection":[
                            {
                                "querySql":[
                                    "select * from ORDER_DETAIL_BAK"
                                ],
                                "jdbcUrl":[
                                    "jdbc:oracle:thin:@192.168.0.197:1521:orcl"
                                ]
                            }
                        ]
                    }
                },
                "writer":{
                    "name":"streamwriter",
                    "parameter":{
                        "visible":true,
                        "encoding":"UTF-8"
                    }
                }
            }
        ]
    }
}

5.sqlserver to mysql

{
  "job": {
    "content": [
      {
        "reader": { //读入库配置，比如说是sql server "name": "", //数据源名称，别瞎写 "parameter": {} //数据库配置信息 }, "writer": { //写入库配置，比如说是mysql "name": "", //数据源名称，别瞎写 "parameter": {} //数据库配置信息 } } ], "setting": { //基本设置 "speed": { //流量控制 "channel": 1, //同步时候的并发数 "byte": 104857600 //同步时候的字节大小，影响速度 }, "errorLimit": { //脏数据控制 "record": 10, //脏数据最大记录数阈值 "percentage": 0.05 //脏数据占比阈值 } } } }