陆续归纳,整理
1.hive优化
1.1建表优化
建表语句demo,存储格式为ORC,并且设置索引,*“orc.bloom.filter.columns”=“distinct_id”,*这一句的意思是 把 distinct_id这一列作为索引。分隔符为 \27,有效防止导入数据的时候,发生数据错位。
参考链接:ORC原理及查询优化
create table if not exists test.cust_events(
event string,
user_id bigint,
distinct_id string,
record_time timestamp,
credit_billing_source string,
transfer_account string,
)
partitioned by ( load_dt VARCHAR(10))
ROW FORMAT DELIMITED FIELDS TERMINATED BY '\27'
STORED AS ORCFile TBLPROPERTIES
('orc.compress'='SNAPPY',
'orc.create.index'='true',
"orc.bloom.filter.columns"="distinct_id",
'orc.bloom.filter.fpp'='0.05',
'orc.stripe.size'='10485760',
'orc.row.index.stride'='10000');
2.使用java连接redis哨兵集群
连接哨兵,下面代码中的ip1,ip2,ip3,port1,port2,port3,请根据自己的环境进行配置
import org.apache.commons.pool2.impl.GenericObjectPoolConfig;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisSentinelPool;
import java.util.HashSet;
import java.util.Set;
public class SentinelPool {
private static JedisSentinelPool pool;
private static Set<String> sentinel = new HashSet<>();
static {
sentinel.add("ip1:port1");
sentinel.add("ip2:port2");
sentinel.add("ip3:port3");
initPool();
}
private static void initPool() {
pool = new JedisSentinelPool("mymaster",
sentinel,
new GenericObjectPoolConfig(),
3000);
}
public static Jedis getJedis() {
return pool.getResource();
}
public static void returnBrokenResource(Jedis jedis) {
pool.returnBrokenResource(jedis);
}
public static void returnResource(Jedis jedis) {
pool.returnResource(jedis);
}
}
对redis进行操作的工具类
import redis.clients.jedis.Jedis;
public class JedisSentinelUtil {
/**
* 设置 key
* @param key 键
* @param value 值
* @return
*/
public static String set(String key, String value) {
Jedis jedis = null;
String result = null;
try {
jedis = SentinelPool.getJedis();
result= jedis.set(key, value);
} catch (Exception e) {
SentinelPool.returnBrokenResource(jedis);
return result;
}
SentinelPool.returnResource(jedis);
return result;
}
/**
* 设置一个有过期时间的 kv
* @param key
* @param value
* @param exTime
* @return
*/
public static String setEx(String key, String value,int exTime) {
Jedis jedis = null;
String result = null;
try {
jedis = SentinelPool.getJedis();
result = jedis.setex(key, exTime, value);
} catch (Exception e) {
SentinelPool.returnBrokenResource(jedis);
return result;
}
SentinelPool.returnResource(jedis);
return result;
}
/**
* key 不存在就设置
* @param key
* @param value
* @return
*/
public static Long setNx(String key, String value) {
Jedis jedis = null;
Long result = null;
try {
jedis = SentinelPool.getJedis();
result= jedis.setnx(key, value);
} catch (Exception e) {
SentinelPool.returnBrokenResource(jedis);
return result;
}
SentinelPool.returnResource(jedis);
return result;
}
/**
* 删除 key
* @param key
* @return
*/
public static Long del(String key) {
Jedis jedis = null;
Long result = null;
try {
jedis = SentinelPool.getJedis();
result = jedis.del(key);
} catch (Exception e) {
SentinelPool.returnBrokenResource(jedis);
return result;
}
SentinelPool.returnResource(jedis);
return result;
}
/**
* 获取 Key 对应的值
* @param key
* @return
*/
public static String get(String key) {
Jedis jedis = null;
String result = null;
try {
jedis = SentinelPool.getJedis();
result = jedis.get(key);
} catch (Exception e) {
SentinelPool.returnBrokenResource(jedis);
return result;
}
SentinelPool.returnResource(jedis);
return result;
}
/**
* 设置一个新的值,并返回旧的值
* @param key
* @param value
* @return
*/
public static String geSet(String key,String value) {
Jedis jedis = null;
String result = null;
try {
jedis = SentinelPool.getJedis();
result = jedis.getSet(key, value);
} catch (Exception e) {
SentinelPool.returnBrokenResource(jedis);
return result;
}
SentinelPool.returnResource(jedis);
return result;
}
/**
* 设置过期时间
* @param key
* @param exTime
* @return
*/
public static Long expire(String key, int exTime) {
Jedis jedis = null;
Long result = null;
try {
jedis = SentinelPool.getJedis();
result= jedis.expire(key, exTime);
} catch (Exception e) {
SentinelPool.returnBrokenResource(jedis);
return result;
}
SentinelPool.returnResource(jedis);
return result;
}
/**
* 获取 dbsize
* @return
*/
public static Long getDbSize() {
Jedis jedis = null;
Long result = null;
try {
jedis = SentinelPool.getJedis();
result = jedis.dbSize();
} catch (Exception e) {
SentinelPool.returnBrokenResource(jedis);
return result;
}
SentinelPool.returnResource(jedis);
return result;
}
}