dataX_from_mysql_to_hive orc分区表

1. hive建表：

CREATE TABLE ods.ods_mysql_to_hive_orc_partiton(

  `id` int ,

  `name` string COMMENT 'name'

)  COMMENT '测试表_测试datax_导入_orc分区表' PARTITIONED BY (`dayid` string comment '分区字段')

ROW FORMAT DELIMITED FIELDS TERMINATED BY '^'  STORED AS orc;

2.导入前，增加分区

alter table ods.ods_mysql_to_hive_orc_partiton drop partition (dayid=${dayid_1d_ago_global})

alter table ods.ods_mysql_to_hive_orc_partiton add partition (dayid=${dayid_1d_ago_global})

3.datax导入语句

{

  "job": {

    "setting": {

      "speed": {

        "channel": "1"

      },

      "errorLimit": {

        "record": 0,

        "percentage": 1

      }

    },

    "content": [

      {

        "reader": {

          "name": "mysqlreader",

          "parameter": {

            "username": "username",

            "password": "password",

            "splitPk": "",

            "column": [

              "`id`","`name`"

            ],

            "where": "",

            "connection": [

              {

                "jdbcUrl": ["jdbc:mysql://192.168.2.149:3306/database_name?serverTimezone=Asia/Shanghai"],

                "table": ["`bable_name`"]

              }

            ]

          }

        },

        "writer": {

          "name": "hdfswriter",

          "parameter": {

            "defaultFS": "hdfs://namenode.hive.com.cn:3366",

            "fileType": "orc",

            "path": "/warehouse/tablespace/managed/hive/ods.db/ods_mysql_to_hive_orc_partiton/dayid=$dayid/",

            "fileName": "000000_0",

            "column": [

              {"name": "id","type": "int"},{"name": "name","type": "string"}

            ],

            "writeMode": "append",

            "fieldDelimiter": "^",

            "compress": "snappy"

          }

        }

      }

    ]

  }

}