Redis集群api和集群状态下如何使用pipeline

0.准备工作 pom文件

        <dependency>
            <groupId>redis.clients</groupId>
            <artifactId>jedis</artifactId>
            <version>2.9.0</version>
        </dependency>

        <dependency>
            <groupId>org.apache.commons</groupId>
            <artifactId>commons-collections4</artifactId>
            <version>4.0</version>
        </dependency>

        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>4.12</version>
        </dependency>

1.使用静态内部类创建单例的jedisCluster连接池工具类

package com.roger.redis.cluster;

import redis.clients.jedis.HostAndPort;
import redis.clients.jedis.JedisCluster;
import redis.clients.jedis.JedisPoolConfig;

import java.util.HashSet;
import java.util.Set;

public class JedisClusterUtil {


    public static JedisCluster getJedisClusterInstance(){
        return SingletonJedisCluster.INSTANCE;
    }

    private static class SingletonJedisCluster{

        private static JedisCluster INSTANCE = initJedisCluster();

       public static JedisCluster initJedisCluster(){
           //redis 节点信息
           Set<HostAndPort> clusterNodes = new HashSet<>();

           initClusterNodes(clusterNodes,"192.168.1.11:8001");
           initClusterNodes(clusterNodes,"192.168.1.11:8004");

           initClusterNodes(clusterNodes,"192.168.1.12:8002");
           initClusterNodes(clusterNodes,"192.168.1.12:8005");

           initClusterNodes(clusterNodes,"192.168.1.13:8003");
           initClusterNodes(clusterNodes,"192.168.1.13:8006");


           JedisCluster jedisCluster = new JedisCluster(clusterNodes,3000,3000,10,"roger",initJedisPoolConfig());
           return jedisCluster;
       }
    }

    private static void initClusterNodes(Set<HostAndPort> clusterNodes,String hostAndPortStr){
        HostAndPort hostAndPort = HostAndPort.parseString(hostAndPortStr);
        clusterNodes.add(hostAndPort);
    }

    private static JedisPoolConfig initJedisPoolConfig(){
        JedisPoolConfig jedisPoolConfig = new JedisPoolConfig();
        //jedis连接池中最大的连接个数
        jedisPoolConfig.setMaxTotal(60);
        //jedis连接池中最大的空闲连接个数
        jedisPoolConfig.setMaxIdle(30);
        //jedis连接池中最小的空闲连接个数
        jedisPoolConfig.setMinIdle(5);
        //jedis连接池最大的等待连接时间 ms值
        jedisPoolConfig.setMaxWaitMillis(3000);

        return jedisPoolConfig;
    }

}

2.创建一个反射工具类，根据对象和域名获取对象的属性信息

package com.roger.redis.utils;

import java.lang.reflect.Field;

public class ReflectUtil {

    public static Field getField(Class<?> clazz,String fieldName){
        try {
            return clazz.getDeclaredField(fieldName);
        } catch (NoSuchFieldException e) {
            if(!clazz.getSuperclass().isAssignableFrom(Object.class)){
                return getField(clazz.getSuperclass(),fieldName);
            }
            return null;
        }
    }

    public static <T> T getObject(Object obj,String fieldName){
        try {
            Field field = getField(obj.getClass(),fieldName);
            field.setAccessible(true);
            return (T) field.get(obj);
        } catch (IllegalAccessException e) {
            return null;
        }
    }

}

3. jedis集群pipeline工具类

package com.roger.redis.cluster.pipeline;

import com.roger.redis.cluster.JedisClusterUtil;
import com.roger.redis.utils.ReflectUtil;
import org.apache.commons.collections4.CollectionUtils;
import redis.clients.jedis.*;
import redis.clients.jedis.exceptions.JedisMovedDataException;
import redis.clients.jedis.exceptions.JedisRedirectionException;
import redis.clients.util.JedisClusterCRC16;
import redis.clients.util.SafeEncoder;

import java.io.Closeable;
import java.util.*;

public class JedisClusterPipeLine extends PipelineBase implements Closeable {

    //一次pipeline过程中使用到jedis缓存
    private final Map<JedisPool,Jedis> poolToJedisMap = new HashMap<>();
    // JedisClusterInfoCache中封装了每个节点对应的Jedis操作客户端
    // 还有槽点对应的Jedis操作客户端
    private final JedisClusterInfoCache clusterInfoCache;
    private final JedisSlotBasedConnectionHandler connectionHandler;
    // 根据顺序存储每个命令对应的Client
    private Queue<Client> clients = new LinkedList<Client>();
    private boolean hasDataInBuff = false;   // 是否有数据在缓存区
    /**
     * 根据jedisCluster实例生成对应的JedisClusterPipeline
     */
    public JedisClusterPipeLine() {
        JedisCluster jedisCluster = JedisClusterUtil.getJedisClusterInstance();
        //根据jedisCluster实例获取其父类BinaryJedisCluster的属性值
        connectionHandler = ReflectUtil.getObject(jedisCluster,"connectionHandler");
        //根据刚刚从jedisCluster实例获取的connectionHandler实例
        // 获取其父类JedisClusterConnectionHandler的属性值
        clusterInfoCache = ReflectUtil.getObject(connectionHandler,"cache");
    }

    /**
     * 由于集群模式存在节点的动态添加删除，且client不能实时感知（只有在执行命令时才可能知道集群发生变更），
     * 因此，该实现不保证一定成功，建议在批量操作之前调用 refreshCluster() 方法重新获取集群信息。
     * 也可以定时刷新以便获取最新的集群信息
     * 还有可以在捕获异常中刷新集群节点信息，重试操作，设置重试次数，不能无线重试
     */
    public void refreshClusterInfo(){
        connectionHandler.renewSlotCache();
    }

    @Override
    protected Client getClient(String key) {
        byte[] bKey = SafeEncoder.encode(key);

        return getClient(bKey);
    }

    @Override
    protected Client getClient(byte[] bkey) {
        //1.根据key值计算槽点
        Jedis jedis = getJedis(JedisClusterCRC16.getSlot(bkey));

        Client client = jedis.getClient();
        clients.add(client);

        return client;
    }

    private Jedis getJedis(int slot) {
        //2.同一个节点下的所有槽点获取到的JedisPool都是一样的
        JedisPool slotJedisPool = clusterInfoCache.getSlotPool(slot);

        //3.验证分配到同一个节点的所有槽点对应的JedisPool是否已经存在
        Jedis jedis = poolToJedisMap.get(slotJedisPool);
        //4. 如果不存在
        if(jedis == null){
            //4.1 则获取客户端连接，存储到容器中去
            jedis = slotJedisPool.getResource();
            poolToJedisMap.put(slotJedisPool,jedis);
        }
        //4.2 如果已经存储直接返回
        hasDataInBuff = true;
        return jedis;
    }


    @Override
    public void close()  {
        clean();
        //清除客户端队列
        clients.clear();
        //关闭缓存的jedis连接
        for (Jedis jedis : poolToJedisMap.values()) {
            if (hasDataInBuff) {
                flushCachedData(jedis);
            }
            jedis.close();
        }
        //清除jedis连接缓存数据
        poolToJedisMap.clear();
        //还原初始状态
        hasDataInBuff = false;
    }

    private void flushCachedData(Jedis jedis) {
        try {
            jedis.getClient().getAll();
        } catch (RuntimeException ex) {
            //捕获异常，保证代码的高可用
        }
    }

    /**
     * 同步读取所有数据. 与syncAndReturnAll()相比，sync()只是没有对数据做反序列化
     */
    public void sync() {
        innerSync(null);
    }

    /**
     * 同步读取所有数据 并按命令顺序返回一个列表
     *
     * @return 按照命令的顺序返回所有的数据
     */
    public List<Object> syncAndReturnAll() {
        List<Object> responseList = new ArrayList<Object>();

        innerSync(responseList);

        return responseList;
    }

    private void innerSync(List<Object> responseList){
        if(responseList == null){
            responseList = new ArrayList<>();
        }

        HashSet<Client> clientSet = new HashSet<Client>();

        try {
            for(Client client : clients){
                // 在sync()调用时其实是不需要解析结果数据的，
                // 但是如果不调用get方法，发生了JedisMovedDataException这样的错误应用是不知道的，
                // 因此需要调用get()来触发错误。
                // 其实如果Response的data属性可以直接获取，可以省掉解析数据的时间，
                // 然而它并没有提供对应方法，要获取data属性就得用反射，不想再反射了，所以就这样了
                Object response = generateResponse(client.getOne()).get();
                if(response != null){
                    responseList.add(response);
                }
                // size相同说明所有的client都已经添加，就不用再调用add方法了
                if (clientSet.size() != poolToJedisMap.size()) {
                    clientSet.add(client);
                }
            }
        } catch (JedisRedirectionException ex) {
            if(ex instanceof JedisMovedDataException){
                // if MOVED redirection occurred, rebuilds cluster's slot cache,
                // recommended by Redis cluster specification
                refreshClusterInfo();
            }
        } finally {
            if (clientSet.size() != poolToJedisMap.size()) {
                // 所有还没有执行过的client要保证执行(flush)，防止放回连接池后后面的命令被污染
                for (Jedis jedis : poolToJedisMap.values()) {
                    if (clientSet.contains(jedis.getClient())) {
                        continue;
                    }

                    flushCachedData(jedis);
                }
            }

            hasDataInBuff = false;
            close();
        }
    }

}

4.junit测试类

‘   1.通过pipeline命令

package com.roger.redis.cluster.pipeline;

import org.junit.Before;
import org.junit.Test;

import java.util.List;

import static org.junit.Assert.*;

public class JedisClusterPipeLineTest {

    private JedisClusterPipeLine jedisClusterPipeLine;

    @Before
    public void setUp(){
        jedisClusterPipeLine = new JedisClusterPipeLine();
    }

    @Test
    public void testJedisClusterPipeLine() {
        long startTime = System.currentTimeMillis();
        //获取最新的集群节点信息
        jedisClusterPipeLine.refreshClusterInfo();

        List<Object> batchResult = null;

        try {
            // batch write
            for (int i = 0; i < 10000; i++) {
                jedisClusterPipeLine.set("k" + i, "v1" + i);
            }
            jedisClusterPipeLine.sync();

            // batch read
            for (int i = 0; i < 10000; i++) {
                jedisClusterPipeLine.get("k" + i);
            }
            batchResult = jedisClusterPipeLine.syncAndReturnAll();
        } finally {
            jedisClusterPipeLine.close();
        }

        long endTime = System.currentTimeMillis();

        System.out.println("总共耗时：" + (endTime - startTime) + "ms; " + (endTime-startTime)/1000 + "s");
        System.out.println("结果集数量" + batchResult.size());
    }
}

      2.不通过pipe命令

package com.roger.redis.cluster;

import org.junit.Before;
import org.junit.Test;
import redis.clients.jedis.JedisCluster;

import java.util.List;

public class JedisClusterUtilTest {

    private JedisCluster jedisCluster;

    @Before
    public void setUp() {
        jedisCluster = JedisClusterUtil.getJedisClusterInstance();
    }

    @Test
    public void testJedisCluster() {
        long startTime = System.currentTimeMillis();

        List<Object> batchResult = new ArrayList<>();

        // batch write
        for (int i = 0; i < 10000; i++) {
            jedisCluster.set("k" + i, "v1" + i);
        }

        // batch read
        for (int i = 0; i < 10000; i++) {
            batchResult.add(jedisCluster.get("k" + i));
        }


        long endTime = System.currentTimeMillis();

        System.out.println("总共耗时：" + (endTime - startTime) + "ms; " + (endTime - startTime) / 1000 + "s");
        System.out.println("结果集数量" + batchResult.size());
    }
}

5.JedisClusterPipeLine实现原理

    5.1) JedisCluster相关类图

    5.2) getClient() 方法的原理

           5.2.1) 根据key值获取到槽点

           5.2.2) 根据槽点获取到JedisPool，又因为同一个节点下所有槽点都对应一个JedisPool

           5.2.3) 定义一个Map<JedisPool,Jedis> poolToJedisMap；含义给批量存储的key值按照槽点分离，

因为5.2.2)获取到的JedisPool作为key存储到poolToJedisMap容器中去，如果已经存在，则不在存储，如果没有

则把5.2.2)获取到的JedisPool作为key存储起来