0073-jedis哨兵模式下感知主节点变更

参考

1. 哨兵模式的客户端连接

哨兵模式下，客户端不能直接连接master节点，需要连接哨兵集群

/**
 * 原始的jedis
 * 1. 可以实现主节点的自动切换
 * 2. 但是池化对象都是master节点，无法实现读写分离
 */
public class Original {
    public static void main(String[] args) {
        // 哨兵集群
        Set<String> sentinels = new HashSet<>(Arrays.asList(
                "centos0010:26379",
                "centos0020:26379",
                "centos0030:26379"
        ));

        JedisPoolConfig jedisPoolConfig = new JedisPoolConfig();
        jedisPoolConfig.setMaxTotal(10);
        jedisPoolConfig.setMaxIdle(5);
        jedisPoolConfig.setMinIdle(5);

        new JedisSentinelPool("mymaster", sentinels, jedisPoolConfig);
        try {
            System.in.read();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}

通过传入的哨兵集群获取主节点的信息

// 类JedisSentinelPool
private HostAndPort initSentinels(Set<String> sentinels, final String masterName) {

    HostAndPort master = null;
    boolean sentinelAvailable = false;

    log.info("Trying to find master from available Sentinels...");
    // 根据哨兵获取主节点信息
    for (String sentinel : sentinels) {
      final HostAndPort hap = HostAndPort.parseString(sentinel);

      log.fine("Connecting to Sentinel " + hap);

      Jedis jedis = null;
      try {
        jedis = new Jedis(hap.getHost(), hap.getPort());
        // 向哨兵发送指令：sentinel get-master-addr-by-name mymaster 可以获取主节点的host的端口
        List<String> masterAddr = jedis.sentinelGetMasterAddrByName(masterName);

        // connected to sentinel...
        sentinelAvailable = true;

        if (masterAddr == null || masterAddr.size() != 2) {
          log.warning("Can not get master addr, master name: " + masterName + ". Sentinel: " + hap
              + ".");
          continue;
        }

        master = toHostAndPort(masterAddr);
        log.fine("Found Redis master at " + master);
        break;
      } catch (JedisException e) {
        // resolves #1036, it should handle JedisException there's another chance
        // of raising JedisDataException
        log.warning("Cannot get master address from sentinel running @ " + hap + ". Reason: " + e
            + ". Trying next one.");
      } finally {
        if (jedis != null) {
          jedis.close();
        }
      }
    }

    if (master == null) {
      if (sentinelAvailable) {
        // can connect to sentinel, but master name seems to not
        // monitored
        throw new JedisException("Can connect to sentinel, but " + masterName
            + " seems to be not monitored...");
      } else {
        throw new JedisConnectionException("All sentinels down, cannot determine where is "
            + masterName + " master is running...");
      }
    }

    log.info("Redis master running at " + master + ", starting Sentinel listeners...");

    // 订阅哨兵频道：+switch-master 获取最新的主节点信息
    for (String sentinel : sentinels) {
      final HostAndPort hap = HostAndPort.parseString(sentinel);
      MasterListener masterListener = new MasterListener(masterName, hap.getHost(), hap.getPort());
      // whether MasterListener threads are alive or not, process can be stopped
      masterListener.setDaemon(true);
      masterListeners.add(masterListener);
      masterListener.start();
    }

    return master;
  }

上面的代码已经拿到了master节点的信息，接下来分析一下哨兵怎么获取新的主节点的

//MasterListener
@Override
    public void run() {

      running.set(true);

      while (running.get()) {

        j = new Jedis(host, port);

        try {
          // double check that it is not being shutdown
          if (!running.get()) {
            break;
          }
        // 订阅哨兵服务器的+switch-master频道，主节点更换以后，新的主节点信息在这里可以获取到
          j.subscribe(new JedisPubSub() {
            @Override
            public void onMessage(String channel, String message) {
              log.fine("Sentinel " + host + ":" + port + " published: " + message + ".");

              String[] switchMasterMsg = message.split(" ");

              if (switchMasterMsg.length > 3) {

                if (masterName.equals(switchMasterMsg[0])) {
                 // 用新主节点信息，构造连接池 initPool(toHostAndPort(Arrays.asList(switchMasterMsg[3], switchMasterMsg[4])));
                } else {
                  log.fine("Ignoring message on +switch-master for master name "
                      + switchMasterMsg[0] + ", our master name is " + masterName);
                }

              } else {
                log.severe("Invalid message received on Sentinel " + host + ":" + port
                    + " on channel +switch-master: " + message);
              }
            }
          }, "+switch-master");

        } catch (JedisConnectionException e) {

          if (running.get()) {
            log.log(Level.SEVERE, "Lost connection to Sentinel at " + host + ":" + port
                + ". Sleeping 5000ms and retrying.", e);
            try {
              Thread.sleep(subscribeRetryWaitTimeMillis);
            } catch (InterruptedException e1) {
              log.log(Level.SEVERE, "Sleep interrupted: ", e1);
            }
          } else {
            log.fine("Unsubscribing from Sentinel at " + host + ":" + port);
          }
        } finally {
          j.close();
        }
      }
    }

启动main观察日志

三月 18, 2020 9:41:11 下午 redis.clients.jedis.JedisSentinelPool initSentinels
信息: Trying to find master from available Sentinels...
三月 18, 2020 9:41:11 下午 redis.clients.jedis.JedisSentinelPool initSentinels
信息: Redis master running at 192.168.247.130:6379, starting Sentinel listeners...
三月 18, 2020 9:41:11 下午 redis.clients.jedis.JedisSentinelPool initPool
信息: Created JedisPool to master at 192.168.247.130:6379

开始启动时，主节点时130，关掉130的redis服务，过一小会儿，就可以自动切换到新的master。

三月 18, 2020 9:41:11 下午 redis.clients.jedis.JedisSentinelPool initSentinels
信息: Trying to find master from available Sentinels...
三月 18, 2020 9:41:11 下午 redis.clients.jedis.JedisSentinelPool initSentinels
信息: Redis master running at 192.168.247.130:6379, starting Sentinel listeners...
三月 18, 2020 9:41:11 下午 redis.clients.jedis.JedisSentinelPool initPool
信息: Created JedisPool to master at 192.168.247.130:6379
三月 18, 2020 9:42:05 下午 redis.clients.jedis.JedisSentinelPool initPool
信息: Created JedisPool to master at 192.168.247.131:6379

2. 存在的不足

上述的模式可以保证集群在切换主节点时客户端可以自动切换到新的主节点，但是连接池中始终都是master的连接，也就是说读写走的都是master，我们可以优化一下，让写走master，读走slave

3. 读写分离

连接池默认获取到的时master的连接，我们可以仿造这个新构建slave的连接池，让系统同时存在两个连接池，一个master的连接池，一个slave的连接池。读写分别取用不同的连接，或者spring中注入不通的连接。

3.1 改造JedisSentinelPool

拷贝JedisSentinelPool，起名为JedisSlaveSentinelPool,修改initPool方法(sentinels,masterName两个变量提升为成员变量)其它不动

private Set<String> sentinels;
private String masterName;

private void initPool(HostAndPort master) {
    if (!master.equals(currentHostMaster)) {
        currentHostMaster = master;
        if (factory == null) {
        // 传入哨兵信息和主节点名称
            factory = new JedisSlaveFactory(sentinels, masterName, connectionTimeout,
                    soTimeout, password, database, clientName, false, null, null, null);
            initPool(poolConfig, factory);
        } else {
            internalPool.clear();
        }

        log.info("Created JedisPool to master at " + master);
    }
}

@Override
public Jedis getResource() {
    while (true) {
        Jedis jedis = super.getResource();
        jedis.setDataSource(this);

        // get a reference because it can change concurrently
        final HostAndPort master = currentHostMaster;
        final HostAndPort connection = new HostAndPort(jedis.getClient().getHost(), jedis.getClient()
                .getPort());

//            if (master.equals(connection)) {
//                // connected to the correct master
//                return jedis;
//            } else {
//                returnBrokenResource(jedis);
//            }
        return jedis;
    }
}

3.2 新建slave的工厂

首先根据masterName获取到slave节点的信息（哨兵客户端执行：sentinel slaves mymaster可以获取slave节点信息），其次根据有效的客户端随机建立连接放入连接池。

package com.honor.remould;

import org.apache.commons.pool2.PooledObject;
import org.apache.commons.pool2.PooledObjectFactory;
import org.apache.commons.pool2.impl.DefaultPooledObject;
import redis.clients.jedis.BinaryJedis;
import redis.clients.jedis.HostAndPort;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.exceptions.JedisConnectionException;
import redis.clients.jedis.exceptions.JedisException;

import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.SSLParameters;
import javax.net.ssl.SSLSocketFactory;
import java.util.*;
import java.util.logging.Logger;

public class JedisSlaveFactory implements PooledObjectFactory<Jedis> {

    protected Logger log = Logger.getLogger(getClass().getName());

    private final Set<String> sentinels;
    private final String masterName;
    private final int connectionTimeout;
    private final int soTimeout;
    private final String password;
    private final int database;
    private final String clientName;
    private final boolean ssl;
    private final SSLSocketFactory sslSocketFactory;
    private SSLParameters sslParameters;
    private HostnameVerifier hostnameVerifier;
    private Random random;

    public JedisSlaveFactory(final Set<String> sentinels, final String masterName, final int connectionTimeout,
                             final int soTimeout, final String password, final int database, final String clientName,
                             final boolean ssl, final SSLSocketFactory sslSocketFactory, final SSLParameters sslParameters,
                             final HostnameVerifier hostnameVerifier) {
        this.sentinels = sentinels;
        this.masterName = masterName;
        this.connectionTimeout = connectionTimeout;
        this.soTimeout = soTimeout;
        this.password = password;
        this.database = database;
        this.clientName = clientName;
        this.ssl = ssl;
        this.sslSocketFactory = sslSocketFactory;
        this.sslParameters = sslParameters;
        this.hostnameVerifier = hostnameVerifier;
        this.random = new Random();
    }

    @Override
    public void activateObject(PooledObject<Jedis> pooledJedis) throws Exception {
        final BinaryJedis jedis = pooledJedis.getObject();
        if (jedis.getDB() != database) {
            jedis.select(database);
        }
    }

    /**
     * 销毁redis底层连接
     */
    @Override
    public void destroyObject(PooledObject<Jedis> pooledJedis) {
        log.info("destroyObject =" + pooledJedis.getObject());
        final BinaryJedis jedis = pooledJedis.getObject();
        if (jedis.isConnected()) {
            try {
                jedis.quit();
                jedis.disconnect();
            } catch (Exception e) {
            }
        }
    }

    /**
     * 创建Redis底层连接对象，返回池化对象.
     */
    @Override
    public PooledObject<Jedis> makeObject() {
        List<HostAndPort> slaves = this.getAlivedSlaves();

        //在slave节点中随机选取一个节点进行连接
        int index = slaves.size() == 1 ? 0 : random.nextInt(slaves.size());
        final HostAndPort hostAndPort = slaves.get(index);

        log.info("Create jedis instance from slaves=[" + slaves + "] , choose=[" + hostAndPort + "]");

        //创建redis客户端
        final Jedis jedis = new Jedis(hostAndPort.getHost(), hostAndPort.getPort(), connectionTimeout,
                soTimeout, ssl, sslSocketFactory, sslParameters, hostnameVerifier);

        //测试连接,设置密码,数据库.
        try {
            jedis.connect();
            if (null != this.password) {
                jedis.auth(this.password);
            }
            if (database != 0) {
                jedis.select(database);
            }
            if (clientName != null) {
                jedis.clientSetname(clientName);
            }
        } catch (JedisException je) {
            jedis.close();
            throw je;
        }

        return new DefaultPooledObject<Jedis>(jedis);
    }


    /**
     * 获取可用的RedisSlave节点信息
     */
    private List<HostAndPort> getAlivedSlaves() {
        log.info("Get alived salves start...");

        List<HostAndPort> alivedSalaves = new ArrayList<>();
        boolean sentinelAvailable = false;

        //循环哨兵,建立连接获取slave节点信息
        //当某个哨兵连接失败，会忽略异常连接下一个哨兵
        for (String sentinel : sentinels) {
            final HostAndPort hap = HostAndPort.parseString(sentinel);

            log.info("Connecting to Sentinel " + hap);

            Jedis jedis = null;
            try {
                jedis = new Jedis(hap.getHost(), hap.getPort());

                List<Map<String, String>> slavesInfo = jedis.sentinelSlaves(masterName);

                //可以连接到哨兵
                sentinelAvailable = true;

                //没有查询到slave信息,循环下一个哨兵
                if (slavesInfo == null || slavesInfo.size() == 0) {
                    log.warning("Cannot get slavesInfo, master name: " + masterName + ". Sentinel: " + hap
                            + ". Trying next one.");
                    continue;
                }

                //获取可用的Slave信息
                for (Map<String, String> slave : slavesInfo) {
                    if (slave.get("flags").equals("slave")) {
                        String host = slave.get("ip");
                        int port = Integer.valueOf(slave.get("port"));
                        HostAndPort hostAndPort = new HostAndPort(host, port);

                        log.info("Found alived redis slave:[" + hostAndPort + "]");

                        alivedSalaves.add(hostAndPort);
                    }
                }

                log.info("Get alived salves end...");
                break;
            } catch (JedisException e) {
                //当前哨兵连接失败,忽略错误连接下一个哨兵
                log.warning("Cannot get slavesInfo from sentinel running @ " + hap + ". Reason: " + e
                        + ". Trying next one.");
            } finally {
                if (jedis != null) {
                    jedis.close();
                }
            }
        }

        //没有可用的slave节点信息
        if (alivedSalaves.isEmpty()) {
            if (sentinelAvailable) {
                throw new JedisException("Can connect to sentinel, but " + masterName
                        + " cannot find any redis slave");
            } else {
                throw new JedisConnectionException("All sentinels down");
            }
        }

        return alivedSalaves;
    }

    @Override
    public void passivateObject(PooledObject<Jedis> pooledJedis) {
    }

    /**
     * 检查jedis客户端是否有效
     *
     * @param pooledJedis 池中对象
     * @return true有效  false无效
     */
    @Override
    public boolean validateObject(PooledObject<Jedis> pooledJedis) {
        final BinaryJedis jedis = pooledJedis.getObject();
        try {
            //是否TCP连接 && 是否ping通  && 是否slave角色
            boolean result = jedis.isConnected()
                    && jedis.ping().equals("PONG")
                    && jedis.info("Replication").contains("role:slave");

            log.info("ValidateObject Jedis=[" + jedis + "] host=[ " + jedis.getClient().getHost() +
                    "] port=[" + jedis.getClient().getPort() + "] return=[" + result + "]");
            return result;
        } catch (final Exception e) {
            log.warning("ValidateObject error jedis client cannot use");
            return false;
        }
    }
}

3.3 用法

用法和建立master的连接池差不多，如下

/**
 * 改造的jredis对象
 * 1. 可以实现主节点自动切换
 * 2. 实现读写分离
 */
public class Remould {
    public static void main(String[] args) {
        JedisPoolConfig jedisPoolConfig = new JedisPoolConfig();
        jedisPoolConfig.setMaxTotal(10);
        jedisPoolConfig.setMaxIdle(1);
        jedisPoolConfig.setMinIdle(1);

        Set<String> sentinels = new HashSet<>(Arrays.asList(
                "centos0010:26379",
                "centos0020:26379",
                "centos0030:26379"
        ));
        // 主节点连接池==>管写
        JedisSentinelPool masterPool = new JedisSentinelPool("mymaster", sentinels, jedisPoolConfig);

        // 从节点连接池==>管读
        JedisSlaveSentinelPool slavePool = new JedisSlaveSentinelPool("mymaster", sentinels, jedisPoolConfig);

        Jedis masterJredis = masterPool.getResource();
        masterJredis.set("hello", "hello, this is sentinel schema");

        Jedis slaveJredis = slavePool.getResource();
        System.out.println(slaveJredis.get("hello"));

        try {
            System.in.read();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}

3.4 spring集成

与Spring集成，分别创建不同的对象即可，在程序中查询接口可以先走slave进行查询，查询不到在查询master， master也没有则写入缓存，返回数据，下载在查询slave就同步过去啦，这样一来redis的性能会大幅度的提升。

@Primary
@Bean(name = "redisTemplateMaster")
public RedisTemplate<Object, Object> redisTemplateMaster() {
    RedisTemplate<Object, Object> template = new RedisTemplate<>();
    template.setConnectionFactory(redisMasterConnectionFactory());
    template.setKeySerializer(new StringRedisSerializer());
    template.setValueSerializer(new GenericJackson2JsonRedisSerializer());
    return template;
}

@Bean(name = "redisTemplateSlave")
public RedisTemplate<Object, Object> redisTemplateSlave() {
    RedisTemplate<Object, Object> template = new RedisTemplate<>();
    template.setConnectionFactory(redisSlaveConnectionFactory());
    template.setKeySerializer(new StringRedisSerializer());
    template.setValueSerializer(new GenericJackson2JsonRedisSerializer());
    return template;
}