hadoop tez hive

终于把tez环境配置好了，所用版本为hadoop2.4，tez0.4.0，hive0.13.1，试了hive0.12，同样的配置不行，无法启动tez的mrr。

本机测试了一下速度，两个表关联测试，不用tez的hadoop2.3用时660秒，这个配置好了用了840秒。也不知道是不是自己的机器不行，放到服务器上再测试了。

最终是看了这个网址才配好的：https://issues.apache.org/jira/browse/HIVE-6098

hive在启动时需要使用：hive -hiveconf hive.execution.engine=tez

启动以后还要运行以下内容（site开头的）：

// needed because SMB isn't supported on tez yet
set hive.optimize.bucketmapjoin=false;
set hive.optimize.bucketmapjoin.sortedmerge=false;
set hive.auto.convert.sortmerge.join=false;
set hive.auto.convert.sortmerge.join.noconditionaltask=false;
set hive.auto.convert.join.noconditionaltask=true;

// depends on your available mem/cluster, but map/reduce mb should be set to the same for container reuse
set hive.auto.convert.join.noconditionaltask.size=64000000;
set mapred.map.child.java.opts=-server -Xmx3584m -Djava.net.preferIPv4Stack=true;
set mapred.reduce.child.java.opts=-server -Xmx3584m -Djava.net.preferIPv4Stack=true;
set mapreduce.map.memory.mb=4096;
set mapreduce.reduce.memory.mb=4096;

// generic opts
set hive.optimize.reducededuplication.min.reducer=1;
set hive.optimize.mapjoin.mapreduce=true;

// autogather might require you to up the max number of counters, if you run into issues
set hive.stats.autogather=true;
set hive.stats.dbclass=counter;

// tea settings can also go into fez-site if desired
set mapreduce.map.output.compress=true;
set mapreduce.map.output.compress.codec=org.apache.hadoop.io.compress.DefaultCodec;
set tez.runtime.intermediate-output.should-compress=true;
set tez.runtime.intermediate-output.compress.codec=org.apache.hadoop.io.compress.DefaultCodec;
set tez.runtime.intermdiate-input.is-compressed=true;
set tez.runtime.intermediate-input.compress.codec=org.apache.hadoop.io.compress.DefaultCodec;

// tez groups in the AM
set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;

set hive.orc.splits.include.file.footer=true;

set hive.root.logger=ERROR,console;
set hive.execution.engine=tez;
set hive.vectorized.execution.enabled=true;
set hive.exec.local.cache=true;
set hive.compute.query.using.stats=true;