datax同步数据：数据源：mysql，目标源：hbase

最新推荐文章于 2024-09-05 23:08:02 发布

原创最新推荐文章于 2024-09-05 23:08:02 发布 · 330 阅读

3 ·

CC 4.0 BY-SA版权

文章标签：

#大数据

datax 专栏收录该内容

6 篇文章

订阅专栏

该配置文件描述了一个从MySQL数据库到HBase的数据迁移作业。使用了MySQLreader读取特定列并基于where条件过滤数据，然后通过hbase11xwriter将数据写入HBase表中。作业设置为单通道执行。

一.配置文件mysql-hbase.json

{
    "job": {
        "content": [
            {
                "reader": {
                    "name": "mysqlreader",
                    "parameter": {
                        "column": ["city_id","cityCode","CityName"],
                        "where":"city_id>100",
                        "connection": [
                            {
                                "jdbcUrl": ["jdbc:mysql://x.x.x.x:3306/dw?characterEncoding=utf-8"],
                                "table": ["dim_city_call"]
                            }
                        ],
                        "password": "xxxxx",
                        "username": "xxx"
                    }
                },
                "writer": {
                      "name": "hbase11xwriter",
                      "parameter": {
                        "hbaseConfig": {
                          "hbase.rootdir": "/hbase",
                          "hbase.cluster.distributed": "true",
                          "hbase.zookeeper.quorum": "cdh3:2181,cdh4:2181,cdh5:2181"
                        },
                        "table": "anguoan",
                        "mode": "normal",
                        "rowkeyColumn": [
                            {"index":0,"type":"string"}
                        ],
                        "column": [
                          {"index":1,"name": "cf1:q1","type": "string"},
                          {"index":2,"name": "cf1:q2","type": "string"}
                        ],
                        "versionColumn":{
                          "index": -1,
                          "value":"123456789"
                        },
                        "encoding": "utf-8"
                      }
                }
            }
        ],
        "setting": {
            "speed": {
                "channel": "1"
            }
        }
    }
}