123.Impala查询缓慢问题与解决

123.1 问题情况

• 前提
  • Impala Daemon已启用-convert_legacy_hive_parquet_utc_timestamps
• Hive创建测试表，其中“statsdate”字段为TIMESTAMP类型：

[root@cdh4 scripts]# cat createSourceTable.sql 
create database if not exists iot_test;
use iot_test;
create table if not exists hive_table_text (
ordercoldaily BIGINT, 
smsusedflow BIGINT, 
gprsusedflow BIGINT, 
statsdate TIMESTAMP, 
custid STRING, 
groupbelong STRING, 
provinceid STRING, 
apn STRING ) 
PARTITIONED BY ( subdir STRING )
ROW FORMAT DELIMITED FIELDS TERMINATED BY "," ;

• 执行命令

./hivesql_exec.sh createSourceTable.sql

• 查看测试表，在beeline中
  
• 执行命令，查看表结构

show create table hive_table_test;

• 测试数据准备
  • 生成测试数据，gendata.sh脚本

[root@cdh4 scripts]# cat gendata.sh 
function rand(){  
    min=$1  
    max=$(($2-$min+1))  
    num=$(($RANDOM+1000000000))
    echo $(($num%$max+$min))  
}  
let i=1
while [ $i -le 3 ];
do
 let n=1
 while [ $n -le $1 ];
 do
  let month=$n%12+1
  if [ $month -eq 2 ];then
    let day=$n%28+1
  else
    let day=$n%30+1
  fi  
  let hour=$n%24
  rnd=$(rand 10000 10100) 
  echo "$i$n,$i$n,$i$n,2017-$month-$day $hour:20:00,${rnd},$n,$n,$n" >> data$i.txt
  let n=n+1
 done
let i=i+1
done

• 执行命令

./gendata.sh 300000