hive开启动态分区，文件压缩

开启动态分区

set hive.exec.dynamic.partition.mode=nonstrict;
set hive.exec.dynamic.partition=true;

开启压缩

set hive.exec.compress.output=true;
set mapred.output.compression.codec=org.apache.hadoop.io.compress.GzipCodec;

开启文件合并默认大小为128M

set hive.merge.tezfiles=true;
set hive.merge.smallfiles.avgsize=128000000;
set hive.merge.size.per.task=128000000;

创建表

CREATE EXTERNAL TABLE `namespace.tableName`(
`字段名称` string COMMENT "描述"
)
PARTITIONED BY ( 
  `ymd` string)
ROW FORMAT DELIMITED 
  FIELDS TERMINATED BY '\u0005' 
STORED AS INPUTFORMAT 
  'org.apache.hadoop.mapred.TextInputFormat' 
OUTPUTFORMAT 
  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'
LOCATION
  'hdfs路径';

查询分区数据写入对应分区

insert into table namespace.tableName partition(分区字段)
select 'sms',mktactivitynbr,phonenum as phone
case sms_result when '0' then '1' else '0' end as code，ymd
from namespace.tableName 
where ymd>=20210104 and ymd<=20210110 and 字段 in('0','1','23') ;

set hive.exec.dynamic.partition.mode=nonstrict;
set hive.exec.dynamic.partition=true;
set hive.exec.compress.output=true;
set mapred.output.compression.codec=org.apache.hadoop.io.compress.GzipCodec;
insert overwrite table dmp_mcc_contact_view partition(ymd) select * from dmp_mcc_contact_view where ymd>=20201127 and ymd<=20201129;