Hive Result Cache -- Hive 查询结果缓存

hive-site.xml 配置内容

<property>
  <name>hive.query.results.cache.enabled</name>
  <value>true</value>
  <description>If the query results cache is enabled. This will keep results of previously executed queries to be reused if the same query is executed again.</description>
</property>
<property>
  <name>hive.query.results.cache.nontransactional.tables.enabled</name>
  <value>false</value>
  <description>If the query results cache is enabled for queries involving non-transactional tables.Users who enable this setting should be willing to tolerate some amount of stale results in the cache.</description>
</property>
<property>
  <name>hive.query.results.cache.wait.for.pending.results</name>
  <value>true</value>
  <description>Should a query wait for the pending results of an already running query, in order to use the cached result when it becomes ready</description>
</property>
<property>
  <name>hive.query.results.cache.directory</name>
  <value>/tmp/hive/_resultscache_</value>
  <description>Location of the query results cache directory. Temporary results from queries will be moved to this location.</description>
</property>
<property>
  <name>hive.query.results.cache.max.entry.lifetime</name>
  <value>3600s</value>
  <description>
    Expects a time value with unit (d/day, h/hour, m/min, s/sec, ms/msec, us/usec, ns/nsec), which is sec if not specified.
    Maximum lifetime in seconds for an entry in the query results cache. A nonpositive value means infinite.
  </description>
</property>
<property>
  <name>hive.query.results.cache.max.size</name>
  <value>2147483648</value>
  <description>Maximum total size in bytes that the query results cache directory is allowed to use on the filesystem.</description>
</property>
<property>
  <name>hive.query.results.cache.max.entry.size</name>
  <value>10485760</value>
  <description>Maximum size in bytes that a single query result is allowed to use in the results cache directory</description>
</property>

事务表测试

create table t1(c1 int) tblproperties('transactional'='true');
insert into t1 values(1),(2),(3);

连续执行两次以下 SQL，可以看到第 1 次执行生成一个任务，第 2 次执行没有。

select c1,count(c1)  from t1 group by c1;

执行一次插入操作。

insert into t1 values(4);

再执行一次查询，可以看到生成一个任务，说明数据改变，缓存失效。

select c1,count(c1)  from t1 group by c1;

非事务表测试

hive.query.results.cache.nontransactional.tables.enabled=false;

set hive.query.results.cache.nontransactional.tables.enabled=false;
create table t2(c1 int);
insert into t2 values(1),(2),(3);

连续执行两次以下 SQL，可以看到都有一个任务运行。

select c1,count(c1)  from t2 group by c1;

hive.query.results.cache.nontransactional.tables.enabled=true;

set hive.query.results.cache.nontransactional.tables.enabled=true;
create table t3(c1 int);
insert into t3 values(1),(2),(3);

连续执行两次以下 SQL，可以看到第 1 次执行生成一个任务，第 2 次执行没有，说明缓存已经起作用。

select c1,count(c1)  from t3 group by c1;

执行一次插入操作。

insert into t3 values(4);

再执行一次查询，可以看到没有生成一个任务，说明数据改变后，非事务表的查询，缓存依然有效。

select c1,count(c1)  from t3 group by c1;