Jedis客户端分片--构建高可用客户端分片

记录是一种精神，是加深理解最好的方式之一。

最近深入研究了Jedis的源码，对Jedis的使用进行深入理解，提笔记录。
曹金桂 cao_jingui@163.com(如有遗漏之处还请指教)
时间：2016年12月06日15:00

概述

分片允许数据存放在不同的物理机器上，以适应数据量过大的场景，克服单台机器内存或者磁盘空间的限制。Redis在3.0之前是没有集群的，就算现在3.0推出3.0集群版本，也没有几家公司用于生产。Jedis作为Java的Redis客户端工具，其内部也提供了客户的分片的实现。具体可以参考我的文章《Jedis使用教程完整版》，讲解了Jedis客户端使用，包括了Jedis的客户端分片的使用方法。

Jedis客户端分片存在问题

Jedis的客户端分片采用一致性hash算法，实现数据的分配存储。但当某一个分片服务挂了之后，客户端需要修改服务的指向才能动态的切换到当前分片的从服务去。其实完全可以采用和JedisSentinel一样的方法，利用Jedis的Sentinel构建高可用的服务。

实现

个人参照JedisSentinelPool的实现，完成了对构建Jedis分片的高可用的修改。源码如下：

package redis.clients.jedis;

import org.apache.commons.pool2.PooledObject;
import org.apache.commons.pool2.PooledObjectFactory;
import org.apache.commons.pool2.impl.DefaultPooledObject;
import org.apache.commons.pool2.impl.GenericObjectPoolConfig;
import redis.clients.jedis.exceptions.JedisConnectionException;
import redis.clients.jedis.exceptions.JedisException;
import redis.clients.util.Hashing;
import redis.clients.util.Pool;

import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Pattern;

public class SentinelShardedJedisPool extends Pool<SentinelShardedJedis> {

    protected Logger log = Logger.getLogger(getClass().getName());

    private final Map<String, JedisShardInfo> shardInfoMap = new ConcurrentHashMap<>(); // key=masterName
    private final Map<String, String> sentinels = new ConcurrentHashMap<>(); // key=host+port value=masterName

    private SentinelShardedJedisFactory factory;
    private Set<MasterListener> masterListeners = new HashSet<>();

    public SentinelShardedJedisPool(Set<SentinelInfo> sentinelInfos, final GenericObjectPoolConfig poolConfig, Hashing algo, Pattern keyTagPattern) {
        initShards(sentinelInfos);
        factory = new SentinelShardedJedisFactory(shardInfoMap.values(), algo, keyTagPattern);
        super.initPool(poolConfig, factory);

    }

    private void initShards(Set<SentinelInfo> sentinelInfos) {
        for (SentinelInfo info : sentinelInfos) {
            JedisShardInfo shardInfo = getJedisSharedInfoFromSentinelInfo(info);
            sentinels.put(shardInfo.getHost() + ":" + shardInfo.getPort(), info.getMasterName());
            shardInfoMap.put(info.getMasterName(), shardInfo);

            for (String sentinel : info.getSentinels()) {
                HostAndPort hap = HostAndPort.parseString(sentinel);
                MasterListener masterListener = new MasterListener(info.getMasterName(), hap.getHost(), hap.getPort());
                masterListeners.add(masterListener);
                masterListener.setDaemon(true);
                masterListener.start();
            }
        }
    }

    private JedisShardInfo getJedisSharedInfoFromSentinelInfo(SentinelInfo info) {
        log.info("Trying to find master from available Sentinels...");
        for (String sentinel : info.getSentinels()) {
            final HostAndPort hap = HostAndPort.parseString(sentinel);
            log.fine("Connecting to Sentinel " + hap);
            Jedis jedis = null;
            try {
                jedis = new Jedis(hap.getHost(), hap.getPort());
                List<String> masterAddr = jedis.sentinelGetMasterAddrByName(info.getMasterName());
                if (masterAddr == null || masterAddr.size() != 2) {
                    log.warning("Can not get master addr, master name: " + info.getMasterName() + ". Sentinel: " + hap + ".");
                    continue;
                }

                String host = masterAddr.get(0);
                int port = Integer.parseInt(masterAddr.get(1));
                JedisShardInfo jedisShardInfo = new JedisShardInfo(host, port);
                log.fine("Found Redis master at " + masterAddr);
                return jedisShardInfo;
            } catch (JedisException e) {
                log.warning("Cannot get master address from sentinel running @ " + hap + ". Reason: " + e + ". Trying next one.");
            } finally {
                if (jedis != null) {
                    jedis.close();
                }
            }
        }
        throw new JedisConnectionException("All sentinels down, cannot determine where is " + info.getMasterName() + " master is running...");
    }

    private void reInitPool() {
        internalPool.clear();
    }

    @Override
    public void destroy() {
        for (MasterListener m : masterListeners) {
            m.shutdown();
        }
        super.destroy();
    }

    @Override
    public SentinelShardedJedis getResource() {
        while (true) {
            SentinelShardedJedis jedis = super.getResource();
            for (Jedis shard : jedis.getAllShards()) {
                String host = shard.getClient().getHost();
                int port = shard.getClient().getPort();
                if (sentinels.containsKey(host + ":" + port)) {
                    jedis.setDataSource(this);
                    return jedis;
                }
            }
            returnBrokenResource(jedis);
        }
    }

    @Override
    protected void returnBrokenResource(final SentinelShardedJedis resource) {
        if (resource != null) {
            returnBrokenResourceObject(resource);
        }
    }

    @Override
    protected void returnResource(final SentinelShardedJedis resource) {
        if (resource != null) {
            resource.resetState();
            returnResourceObject(resource);
        }
    }

    private class MasterListener extends Thread {
        private String masterName;
        private String sentinelHost;
        private int sentinelPort;
        private volatile Jedis j;
        private AtomicBoolean running = new AtomicBoolean(false);

        public MasterListener(String masterName, String host, int port) {
            this.masterName = masterName;
            this.sentinelHost = host;
            this.sentinelPort = port;
            this.setName(String.format("MasterListener-%s-[%s:%d]", masterName, host, port));
        }

        @Override
        public void run() {
            running.set(true);
            while (running.get()) {
                j = new Jedis(sentinelHost, sentinelPort);
                try {
                    if (!running.get()) {
                        break;
                    }
                    j.subscribe(new JedisPubSub() {
                        @Override
                        public void onMessage(String channel, String message) {
                            log.fine("Sentinel " + sentinelHost + ":" + sentinelPort + " published: " + message + ".");
                            String[] switchMasterMsg = message.split(" ");
                            if (switchMasterMsg.length > 3) {
                                if (masterName.equals(switchMasterMsg[0])) {
                                    JedisShardInfo oldInfo = shardInfoMap.get(masterName);
                                    sentinels.remove(oldInfo.getHost() + ":" + oldInfo.getPort());

                                    JedisShardInfo newInfo = new JedisShardInfo(switchMasterMsg[3], switchMasterMsg[4]);
                                    newInfo.setConnectionTimeout(oldInfo.getConnectionTimeout());
                                    newInfo.setPassword(oldInfo.getPassword());
                                    newInfo.setSoTimeout(oldInfo.getSoTimeout());
                                    shardInfoMap.put(masterName, newInfo);
                                    sentinels.put(newInfo.getHost() + ":" + newInfo.getPort(), masterName);
                                    reInitPool();
                                } else {
                                    log.fine("Ignoring message on +switch-master for master name " + switchMasterMsg[0] + ", our master name is " + masterName);
                                }

                            } else {
                                log.severe("Invalid message received on Sentinel " + sentinelHost + ":" + sentinelPort + " on channel +switch-master: " + message);
                            }
                        }
                    }, "+switch-master");
                } catch (JedisConnectionException e) {
                    if (running.get()) {
                        log.log(Level.SEVERE, "Lost connection to Sentinel at " + sentinelHost + ":" + sentinelPort + ". Sleeping 5000ms and retrying.", e);
                        try {
                            Thread.sleep(5000);
                        } catch (InterruptedException e1) {
                            log.log(Level.SEVERE, "Sleep interrupted: ", e1);
                        }
                    } else {
                        log.fine("Unsubscribing from Sentinel at " + sentinelHost + ":" + sentinelPort);
                    }
                } finally {
                    j.close();
                }
            }
        }

        public void shutdown() {
            try {
                log.fine("Shutting down listener on " + sentinelHost + ":" + sentinelPort);
                running.set(false);
                if (j != null) {
                    j.disconnect();
                }
            } catch (Exception e) {
                log.log(Level.SEVERE, "Caught exception while shutting down: ", e);
            }
        }
    }

    /**
     * PoolableObjectFactory custom impl.
     */
    private class SentinelShardedJedisFactory implements PooledObjectFactory<SentinelShardedJedis> {

        private Collection<JedisShardInfo> shards;
        private Hashing algo;
        private Pattern keyTagPattern;

        public SentinelShardedJedisFactory(Collection<JedisShardInfo> shards, Hashing algo, Pattern keyTagPattern) {
            this.shards = shards;
            this.algo = algo;
            this.keyTagPattern = keyTagPattern;
        }

        @Override
        public PooledObject<SentinelShardedJedis> makeObject() throws Exception {
            SentinelShardedJedis jedis = new SentinelShardedJedis(new ArrayList<>(shards), algo, keyTagPattern);
            return new DefaultPooledObject<SentinelShardedJedis>(jedis);
        }

        @Override
        public void destroyObject(PooledObject<SentinelShardedJedis> pooledShardedJedis) throws Exception {
            final SentinelShardedJedis shardedJedis = pooledShardedJedis.getObject();
            for (Jedis jedis : shardedJedis.getAllShards()) {
                try {
                    try {
                        jedis.quit();
                    } catch (Exception e) {

                    }
                    jedis.disconnect();
                } catch (Exception e) {

                }
            }
        }

        @Override
        public boolean validateObject(PooledObject<SentinelShardedJedis> pooledShardedJedis) {
            try {
                SentinelShardedJedis jedis = pooledShardedJedis.getObject();
                for (Jedis shard : jedis.getAllShards()) {
                    String host = shard.getClient().getHost();
                    int port = shard.getClient().getPort();
                    if (!shardInfoMap.containsKey(host + port)) {
                        return false;
                    }

                    if (!shard.ping().equals("PONG")) {
                        return false;
                    }
                }
                return true;
            } catch (Exception ex) {
                return false;
            }
        }

        @Override
        public void activateObject(PooledObject<SentinelShardedJedis> p) throws Exception {

        }

        @Override
        public void passivateObject(PooledObject<SentinelShardedJedis> p) throws Exception {

        }
    }

    public static class SentinelInfo {
        private String masterName;
        private String clientName;
        private Set<String> sentinels;

        public SentinelInfo(String masterName, String clientName, Set<String> sentinels) {
            this.masterName = masterName;
            this.clientName = clientName;
            this.sentinels = sentinels;
        }

        public String getMasterName() {
            return masterName;
        }

        public String getClientName() {
            return clientName;
        }

        public Set<String> getSentinels() {
            return sentinels;
        }
    }
}

package redis.clients.jedis;

import redis.clients.util.Hashing;

import java.util.List;
import java.util.regex.Pattern;

public class SentinelShardedJedis extends ShardedJedis {

    protected SentinelShardedJedisPool dataSource = null;

    public SentinelShardedJedis(List<JedisShardInfo> shards) {
        super(shards);
    }

    public SentinelShardedJedis(List<JedisShardInfo> shards, Hashing algo) {
        super(shards, algo);
    }

    public SentinelShardedJedis(List<JedisShardInfo> shards, Pattern keyTagPattern) {
        super(shards, keyTagPattern);
    }

    public SentinelShardedJedis(List<JedisShardInfo> shards, Hashing algo, Pattern keyTagPattern) {
        super(shards, algo, keyTagPattern);
    }

    public void setDataSource(SentinelShardedJedisPool sentinelShardedJedisPool) {
        this.dataSource = sentinelShardedJedisPool;
    }

}

使用方法

JedisPoolConfig poolConfig = new JedisPoolConfig();
poolConfig.setMinIdle(4);
poolConfig.setMaxTotal(100);
poolConfig.setMaxWaitMillis(30000);

Set<String> sentinels = new HashSet<>();
sentinels.add("192.168.2.122:26379");
sentinels.add("192.168.2.128:26379");
Set<SentinelShardedJedisPool.SentinelInfo> sentinelInfos = new HashSet<>();
sentinelInfos.add(new SentinelShardedJedisPool.SentinelInfo("mymaster1", "m1", sentinels));
sentinelInfos.add(new SentinelShardedJedisPool.SentinelInfo("mymaster2", "m2", sentinels));
SentinelShardedJedisPool pool = new SentinelShardedJedisPool(sentinelInfos, poolConfig, Hashing.MURMUR_HASH, null);
SentinelShardedJedis jedis = pool.getResource();
jedis.set("key", "value");
jedis.close();
pool.close();

小结

本实现个人已经测试通过。但若采用本方案必须要经过一定的性能测试。如果问题欢迎一起讨论。