本文介绍了使用DataX实现从Hive到MySQL的数据同步过程，详细配置包括HDFSreader读取Hive数据，设置字段类型和分隔符，以及MySQLwriter写入数据，包括预清理SQL和插入模式。
{
"core": {
"transport": {
"channel": {
"speed": {
"record": "10000",
"byte": 1048576
}
}
}
},
"job": {
"setting": {
"speed": {
"channel": 2,
"byte":1048576,
"record":10000
}
},
"content": [{
"reader": {
"name": "hdfsreader",
"parameter": {
"path": "/usr/hive/warehouse/reco_train.db/*****/*",
"defaultFS": "hdfs://HDFS*****/",
"column": [{
"index": 0,
"type": "string"
},
{
"index": 1,
"type": "string"
},
{
"index": 2,
"type": "string"
},
{
"index": 3,
"type": "string"
}
],
"fileType": "text",
"encoding": "UTF-8",
"fieldDelimiter": "\u0001",
"hadoopConfig":{
"dfs.nameservices": "HDFS*****",
"dfs.ha.namenodes.HDFS*****": "nn1,nn2",
"dfs.namenode.rpc-address.HDFS*****.nn1": "*****:4007",
"dfs.namenode.rpc-address.HDFS*****.nn2": "*****:4007",
"dfs.client.failover.proxy.provider.HDFS*****": "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"
}
}
},
"writer": {
"name": "mysqlwriter",
"parameter": {
"writeMode": "insert",
"username": "*****",
"password": "*****",
"column": [
"sku",
"one_category",
"two_category",
"three_category"
],
"session": [
"set session sql_mode='ANSI'"
],
"preSql": [
"DELETE FROM *****"
],
"connection": [{
"jdbcUrl": "jdbc:mysql://*****:3306/databasesName?useUnicode=true&characterEncoding=UTF-8",
"table": [
"*****"
]
}]
}
}
}]
}
}
扫一扫
3205
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
