该文描述了如何在Hive中创建一个ORC分区表，包括表结构定义和注释设置。接着，展示了在导入数据前如何使用ALTERTABLE命令管理分区。最后，提供了DataX配置示例，用于从MySQL读取数据并写入到HDFS上的HiveORC表，采用Snappy压缩。
1. hive建表：
CREATE TABLE ods.ods_mysql_to_hive_orc_partiton(
`id` int ,
`name` string COMMENT 'name'
) COMMENT '测试表_测试datax_导入_orc分区表' PARTITIONED BY (`dayid` string comment '分区字段')
ROW FORMAT DELIMITED FIELDS TERMINATED BY '^' STORED AS orc;
2.导入前，增加分区
alter table ods.ods_mysql_to_hive_orc_partiton drop partition (dayid=${dayid_1d_ago_global})
alter table ods.ods_mysql_to_hive_orc_partiton add partition (dayid=${dayid_1d_ago_global})
3.datax导入语句
{
"job": {
"setting": {
"speed": {
"channel": "1"
},
"errorLimit": {
"record": 0,
"percentage": 1
}
},
"content": [
{
"reader": {
"name": "mysqlreader",
"parameter": {
"username": "username",
"password": "password",
"splitPk": "",
"column": [
"`id`","`name`"
],
"where": "",
"connection": [
{
"jdbcUrl": ["jdbc:mysql://192.168.2.149:3306/database_name?serverTimezone=Asia/Shanghai"],
"table": ["`bable_name`"]
}
]
}
},
"writer": {
"name": "hdfswriter",
"parameter": {
"defaultFS": "hdfs://namenode.hive.com.cn:3366",
"fileType": "orc",
"path": "/warehouse/tablespace/managed/hive/ods.db/ods_mysql_to_hive_orc_partiton/dayid=$dayid/",
"fileName": "000000_0",
"column": [
{"name": "id","type": "int"},{"name": "name","type": "string"}
],
"writeMode": "append",
"fieldDelimiter": "^",
"compress": "snappy"
}
}
}
]
}
}
扫一扫
2669
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
