Jedis分片策略-一致性Hash
1. Spring配置文件:配置redis的参数
<bean id="redisUtils" class="com.jd.data.spring.RedisClientFactoryBean">
<!--zookeeper 配置优先文本配置,如果两个都有配置,只从zookeeper中取redis config -->
<!-- 文本配置 开始 -->
<!-- 单个应用中的链接池最大链接数-->
<property name="maxActive" value="${redis.maxActive}"/>
<!-- 单个应用中的链接池最大空闲数-->
<property name="maxIdle" value="${redis.maxIdle}"/>
<!-- 单个应用中的链接池取链接时最大等待时间,单位:ms-->
<property name="maxWait" value="${redis.maxWait}"/>
<!-- 设置在每一次取对象时测试ping-->
<property name="testOnBorrow" value="${testOnBorrow}"/>
<!-- 设置redis connect request response timeout 单位:ms-->
<property name="timeout" value="${redis.timeout}"/>
<!-- master redis server 设置 -->
<!-- host:port:password[可选,password中不要有":"],redis server顺序信息一定不要乱,请按照分配顺序填写,乱了就可能会出现一致性hash不同,造成不命中cache情况-->
<property name="masterConfString" value="${redis.master.hosts}"/>
<!-- slave redis server 设置[可选]-->
<!-- host:port:password[可选,password中不要有":"],redis server顺序信息一定不要乱,请按照分配顺序填写,乱了就可能会出现一致性hash不同,造成不命中cache情况-->
<property name="slaveConfString" value="${redis.slave.hosts}"/>
<!-- 文本配置 结束 -->
<!--zookeeper 配置优先文本配置,如果两个都有,只从zookeeper中取redis config -->
<!-- zookeeper server 地址-->
<property name="zooKeeperServers" value="${redis.zkServers}"/>
<!-- zookeeper中 redis config node path-->
<property name="zooKeeperConfigRedisNodeName" value="${zooKeeperConfigRedisNodeName}"/>
<!-- zookeeper client timeout -->
<property name="zooKeeperTimeout" value="${redis.zkSessionTimeout}"/>
<!--zookeeper 配置结束 -->
</bean>
2. FactoryBean初始化RedisUtil对象
public class RedisClientFactoryBean implements FactoryBean{
private ConnectionFactoryBuilder connectionFactoryBuilder = new ConnectionFactoryBuilder();
private List<String> masterConfList = null;
private List<String> slaveConfList = null;
public Object getObject() throws Exception {
//优先zookeeper配置,先检查,由于是分布式环境,我们在上线前手动调用zk,往一个目录中初始化redis参数,这样之后的线上环境就会读取zk里的redis信息了
if(connectionFactoryBuilder.getZookeeperServers()!=null && connectionFactoryBuilder.getZookeeperServers().trim().length()>0
&& connectionFactoryBuilder.getZookeeperConfigRedisNodeName()!=null && connectionFactoryBuilder.getZookeeperConfigRedisNodeName().trim().length()>0){
return new RedisUtils(connectionFactoryBuilder);
}
//检查spring redis server配置
else if (slaveConfList==null || slaveConfList.size()==0){
return newRedisUtils(connectionFactoryBuilder,masterConfList);
}else if (masterConfList!=null && masterConfList.size()>0 && slaveConfList!=null && slaveConfList.size()>0){
return new RedisUtils(connectionFactoryBuilder,masterConfList,slaveConfList);
}else{
throw new ExceptionInInitializerError("redisUtils all init parameter is empty,please check spring config file!");
}
}
…
}
3. RedisUtil执行init方法
public RedisUtils(ConnectionFactoryBuilder connectionFactoryBuilder, List<String> masterConfList, List<String> slaveConfList) {
this.connectionFactoryBuilder = connectionFactoryBuilder;
this.masterConfList = masterConfList;
this.slaveConfList = slaveConfList;
init();
}
private void init() {
log.info("init start~");
List<JedisShardInfo> wShards = null;
List<JedisShardInfo> rShards = null;
//检查masterConfString 是否设置
if (StringUtils.hasLength(connectionFactoryBuilder.getMasterConfString())) {
//log.info("MasterConfString:" + connectionFactoryBuilder.getMasterConfString());
masterConfList = Arrays.asList(connectionFactoryBuilder.getMasterConfString().split("(?:\\s|,)+"));
}
if (CollectionUtils.isEmpty(this.masterConfList)) {
throw new ExceptionInInitializerError("masterConfString is empty!");
}
wShards = new ArrayList<JedisShardInfo>();
for (String wAddr : this.masterConfList) {
if (wAddr != null) {
String[] wAddrArray = wAddr.split(":");
if (wAddrArray.length == 1) {
throw new ExceptionInInitializerError(wAddr + " is not include host:port or host:port:passwd after split \":\"");
}
String host = wAddrArray[0];
int port = Integer.valueOf(wAddrArray[1]);
JedisShardInfo jedisShardInfo = new JedisShardInfo(host, port, connectionFactoryBuilder.getTimeout());
log.info("masterConfList:" + jedisShardInfo.toString());
//检查密码是否需要设置
if (wAddrArray.length == 3 && StringUtils.hasLength(wAddrArray[2])) {
jedisShardInfo.setPassword(wAddrArray[2]);
}
wShards.add(jedisShardInfo);
}
}
//这里我们控制了读写分离,生成了两个连接池,一个wrtiePool,一个readPool。保存了我们的JedisShardInfo集合。
this.writePool = new ShardedJedisPool(connectionFactoryBuilder.getJedisPoolConfig(), wShards);
//检查slaveConfString 是否设置,并且检查主串与从串是否一致
if (StringUtils.hasLength(connectionFactoryBuilder.getSlaveConfString()) &&
!connectionFactoryBuilder.getSlaveConfString().equals(connectionFactoryBuilder.getMasterConfString())) {
//log.info("SlaveConfString:" + connectionFactoryBuilder.getSlaveConfString());
slaveConfList = Arrays.asList(connectionFactoryBuilder.getSlaveConfString().split("(?:\\s|,)+"));
//检查是否有slave配置
if (!CollectionUtils.isEmpty(this.slaveConfList)) {
rShards = new ArrayList<JedisShardInfo>();
for (String rAddr : this.slaveConfList) {
if (rAddr != null) {
String[] rAddrArray = rAddr.split(":");
if (rAddrArray.length == 1) {
throw new ExceptionInInitializerError(rAddr + " is not include host:port or host:port:passwd after split \":\"");
}
String host = rAddrArray[0];
int port = Integer.valueOf(rAddrArray[1]);
JedisShardInfo jedisShardInfo = new JedisShardInfo(host, port, connectionFactoryBuilder.getTimeout());
//检查密码是否需要设置
if (rAddrArray.length == 3 && StringUtils.hasLength(rAddrArray[2])) {
jedisShardInfo.setPassword(rAddrArray[2]);
}
log.info("slaveConfList:" + jedisShardInfo.toString());
rShards.add(jedisShardInfo);
}
}
this.readPool = new ShardedJedisPool(connectionFactoryBuilder.getJedisPoolConfig(), rShards);
//在开启从机时,错误次数默认为1
this.errorRetryTimes = 1;
}
}
//出错后的重试次数
if (connectionFactoryBuilder.getErrorRetryTimes() > 0) {
this.errorRetryTimes = connectionFactoryBuilder.getErrorRetryTimes();
log.error("after error occured redis api retry times is " + this.errorRetryTimes);
}
//是否有错误重试检查
if (connectionFactoryBuilder.getErrorRetryTimes() > 0 && readPool == null) {
//将主的连接池与从连接池设置为相同,为重试做准备
this.readPool = this.writePool;
log.error("readPool is null and errorRetryTimes >0,readPool is set to writePool");
}
//Object转码类定义
transcoder = connectionFactoryBuilder.getDefaultTranscoder();
log.info("init end~");
}
遍历配置文件中的主从配置信息,构造一个JedisShardInfo对象,我们看下JedisShardInfo对象信息:
public class JedisShardInfo extends ShardInfo<Jedis> {
//包含服务器的配置信息
private int timeout;
private String host;
private int port;
private String password = null;
private String name = null;
//重写createResource方法,用来生成该类对应的Jedis对象
@Override
public Jedis createResource() {
return new Jedis(this);
}
}
public abstract class ShardInfo<T> {
private int weight;//父类中包含了一个重要属性:权重,作为本jedis服务器的权值。
4. 构造ShardedJedis
构造ShardedJedis时,需要传入一个JedisShardInfo列表。然后ShardedJedis的父类的父类即Sharded就会对这个list进行初始化。
public class Sharded<R, S extends ShardInfo<R>> {
public static final int DEFAULT_WEIGHT = 1; //默认权重1
private TreeMap<Long, S> nodes; //一个treeMap,保存虚拟节点,模拟一致性hash
private final Hashing algo;//hash算法,默认是murmurhash,这个算法的随机分布比较好
private final Map<ShardInfo<R>, R> resources = new LinkedHashMap<ShardInfo<R>, R>(); //保存shardInfo和jedis的映射关系,相当于一个主机对应的一个jedis,然后这个jedis来保存我们的缓存信息。
public Sharded(List<S> shards, Hashing algo, Pattern tagPattern) {
this.algo = algo;
this.tagPattern = tagPattern;
initialize(shards); //通过构造方法初始化虚拟节点和主机与jedis的映射关系
}
private void initialize(List<S> shards) {
nodes = new TreeMap<Long, S>();
for (int i = 0; i != shards.size(); ++i) { //遍历分片信息,即我们RedisUtil中初始化的List<JedisShardInfo>
final S shardInfo = shards.get(i);
if (shardInfo.getName() == null)
//一致性hash的核心:每个主机散列成160*权重个虚拟节点,分散在一个treeMap中
for (int n = 0; n < 160 * shardInfo.getWeight(); n++) {
nodes.put(this.algo.hash("SHARD-" + i + "-NODE-" + n), shardInfo);
}
else
for (int n = 0; n < 160 * shardInfo.getWeight(); n++) {
nodes.put(this.algo.hash(shardInfo.getName() + "*" + shardInfo.getWeight() + n), shardInfo);
}
resources.put(shardInfo, shardInfo.createResource());//保存shardinfo与jedis的映射关系
}
}
这里的initialize方法是一致性Hash的核心。把每个主机对应成160*权重个虚拟节点,分散在环上(这里用的是TreeMap),比如有4台主机,权重为1,这样每个主机hash过来的key就可以分散成,160份,而非简单的1份,加大了散列的均匀性,更利于hash的性能。
如果不用虚拟节点,比如只有4个主机,那么只有4个节点,假设node1对应key为0-1000的信息;node2对应1001-2000的信息… 这样,如果来了10个key全都小于1000,那么这些key全都分布在了node1上,node2,node3,node4根本没有key,分布很不均匀。
现在用了虚拟节点,一共有640个node,这样相当于node1-node160 对应key为0-1000的信息,node161-node320对应的key为1001-2000的信息,这样我们来了10个key全都小于1000,我们就会在node1-node160中找到10个node, 由于虚拟节点的treeMap是一颗红黑树,所以节点分布的比较均匀,而这10个node可能覆盖了所有的主机,这样我们的分布就非常均匀了。Weight越大,相当于一致性Hash的环路分布越密集,key对应的主机分散的概率就越大。
5. Set方法
RedisUtil中的方法:
ShardedJedis j = null;
String result = null;
j = writePool.getResource(); //1.从写连接池中获取一个JedisShardInfo,然后获取对应的Jedis对象
result = j.set(key, value); //调用ShardedJedis中的set方法
ShardedJedis的set方法:
public String set(String key, String value) {
Jedis j = getShard(key); //根据key,获取虚拟节点,
return j.set(key, value); //
}
public R getShard(String key) {
return resources.get(getShardInfo(key));//根据key对应的JedisShardInfo信息从上文生成的resouces Map中拿到Jedis对象
}
public S getShardInfo(byte[] key) {
//获取key的hash值key1,然后在虚拟节点的map中找到key大于key1的节点,返回此映射的部分视图,利用map的tailMap方法。
SortedMap<Long, S> tail = nodes.tailMap(algo.hash(key));
if (tail.size() == 0) {
return nodes.get(nodes.firstKey());//获取第一个大于key1的虚拟节点
}
return tail.get(tail.firstKey());
}
大体思路:根据key进行murmurhash获取value,然后根据这个value,去nodes中查找key大于这个value的第一个键值对,返回对应的sharedInfo,(即一致性hash中,查找key对应的后续最近的服务器节点保存。),然后根据返回的sharedInfo从resource中获取对应的Jedis对象,然后进行set。
6. Get方法
RedisUtils :
public String get(String key) throws RedisAccessException {
return get(errorRetryTimes, key); //这里的errorRetryTimes=0
}
private String get(int toTryCount, String key) throws RedisAccessException {
String result = null;
ShardedJedis j = null;
boolean flag = true;
try {
if (toTryCount > 0) {//如果大于0,则读从库
j = readPool.getResource();
} else { //如果不大于0,则读主库
j = writePool.getResource();
}
result = j.get(key); //拿到ShardedJedis对象
…
}
ShardedJedis: get和set的逻辑基本一致了,先获取shard,然后获取jedis,然后获取value
public String get(String key) {
Jedis j = getShard(key);
return j.get(key);
}