HDFS空间维护操作

删除临时目录：

kinit -kt /etc/security/keytabs/xxx.keytab xxx #改为自己的用户

 

hdfs dfs -du -h /user/hive/warehouse/icl_prd.db/ | grep '_temp_'

hdfs dfs -rm -r $path_name

 

压缩datax采集的数据：

进入代理机：20.0.40.222

hdfs dfs -ls -R /user/hive/warehouse/icl_prd.db/ | grep 'DATAX_' | awk '{print $8}'> compress.out

sh a.sh   #输出b.sql

grep stat_dt compress.out > a.out
file_name='a.out'
out_file_name='b.sql'
src_sql="insert overwrite table icl_prd.TABLE_NAME_FOR_REPLACE partition (stat_dt) select \* from icl_prd.TABLE_NAME_FOR_REPLACE where WHERE_CONFIG_FOR_REPLACE;"
tb_name=''
where_config=''

cd ./
rm -f $out_file_name

echo 'set hive.exec.dynamic.partition.mode=nonstrict;' >> $out_file_name
echo 'set hive.exec.max.dynamic.partitions=2048;' >> $out_file_name
echo 'set hive.exec.max.dynamic.partitions.pernode=1024;' >> $out_file_name
echo 'set hive.exec.max.created.files=20000;' >> $out_file_name

#string="/hello/shell/haha"  
#array=(${string })
#echo ${array[0]}
#echo ${array[1]}
#echo ${array[2]}

i=0
cat $file_name | while read line
do
 #echo $line
 array=(${line })
 if [ $i -eq 0 ]
 then 
 	#echo ${array[5]}
 	#echo ${array[6]}
 	echo $src_sql | sed "s/TABLE_NAME_FOR_REPLACE/${array[4]}/g" | sed "s/WHERE_CONFIG_FOR_REPLACE/${array[5]}/g" |sed 's/\\//g' >> $out_file_name
 else 
  #echo ${array[4]}
 	#echo ${array[5]}
 	echo $src_sql | sed "s/TABLE_NAME_FOR_REPLACE/${array[4]}/g" | sed "s/WHERE_CONFIG_FOR_REPLACE/${array[5]}/g" |sed 's/\\//g' >> $out_file_name
 fi
 i=1
done
exit 0

执行b.sql

 

b.sql内容如下：   

set hive.exec.dynamic.partition.mode=nonstrict;

set hive.exec.max.dynamic.partitions=2048;

set hive.exec.max.dynamic.partitions.pernode=1024;

set hive.exec.max.created.files=20000;

insert overwrite table icl_prd.amq_t_case_operation partition (stat_dt)

select * from icl_prd.amq_t_case_operation where stat_dt='20200501';

......

 

 

观察空间变化：