0.准备工作 pom文件
<dependency>
<groupId>redis.clients</groupId>
<artifactId>jedis</artifactId>
<version>2.9.0</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-collections4</artifactId>
<version>4.0</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
</dependency>
1.使用静态内部类创建单例的jedisCluster连接池工具类
package com.roger.redis.cluster;
import redis.clients.jedis.HostAndPort;
import redis.clients.jedis.JedisCluster;
import redis.clients.jedis.JedisPoolConfig;
import java.util.HashSet;
import java.util.Set;
public class JedisClusterUtil {
public static JedisCluster getJedisClusterInstance(){
return SingletonJedisCluster.INSTANCE;
}
private static class SingletonJedisCluster{
private static JedisCluster INSTANCE = initJedisCluster();
public static JedisCluster initJedisCluster(){
//redis 节点信息
Set<HostAndPort> clusterNodes = new HashSet<>();
initClusterNodes(clusterNodes,"192.168.1.11:8001");
initClusterNodes(clusterNodes,"192.168.1.11:8004");
initClusterNodes(clusterNodes,"192.168.1.12:8002");
initClusterNodes(clusterNodes,"192.168.1.12:8005");
initClusterNodes(clusterNodes,"192.168.1.13:8003");
initClusterNodes(clusterNodes,"192.168.1.13:8006");
JedisCluster jedisCluster = new JedisCluster(clusterNodes,3000,3000,10,"roger",initJedisPoolConfig());
return jedisCluster;
}
}
private static void initClusterNodes(Set<HostAndPort> clusterNodes,String hostAndPortStr){
HostAndPort hostAndPort = HostAndPort.parseString(hostAndPortStr);
clusterNodes.add(hostAndPort);
}
private static JedisPoolConfig initJedisPoolConfig(){
JedisPoolConfig jedisPoolConfig = new JedisPoolConfig();
//jedis连接池中最大的连接个数
jedisPoolConfig.setMaxTotal(60);
//jedis连接池中最大的空闲连接个数
jedisPoolConfig.setMaxIdle(30);
//jedis连接池中最小的空闲连接个数
jedisPoolConfig.setMinIdle(5);
//jedis连接池最大的等待连接时间 ms值
jedisPoolConfig.setMaxWaitMillis(3000);
return jedisPoolConfig;
}
}
2.创建一个反射工具类,根据对象和域名获取对象的属性信息
package com.roger.redis.utils;
import java.lang.reflect.Field;
public class ReflectUtil {
public static Field getField(Class<?> clazz,String fieldName){
try {
return clazz.getDeclaredField(fieldName);
} catch (NoSuchFieldException e) {
if(!clazz.getSuperclass().isAssignableFrom(Object.class)){
return getField(clazz.getSuperclass(),fieldName);
}
return null;
}
}
public static <T> T getObject(Object obj,String fieldName){
try {
Field field = getField(obj.getClass(),fieldName);
field.setAccessible(true);
return (T) field.get(obj);
} catch (IllegalAccessException e) {
return null;
}
}
}
3. jedis集群pipeline工具类
package com.roger.redis.cluster.pipeline;
import com.roger.redis.cluster.JedisClusterUtil;
import com.roger.redis.utils.ReflectUtil;
import org.apache.commons.collections4.CollectionUtils;
import redis.clients.jedis.*;
import redis.clients.jedis.exceptions.JedisMovedDataException;
import redis.clients.jedis.exceptions.JedisRedirectionException;
import redis.clients.util.JedisClusterCRC16;
import redis.clients.util.SafeEncoder;
import java.io.Closeable;
import java.util.*;
public class JedisClusterPipeLine extends PipelineBase implements Closeable {
//一次pipeline过程中使用到jedis缓存
private final Map<JedisPool,Jedis> poolToJedisMap = new HashMap<>();
// JedisClusterInfoCache中封装了每个节点对应的Jedis操作客户端
// 还有槽点对应的Jedis操作客户端
private final JedisClusterInfoCache clusterInfoCache;
private final JedisSlotBasedConnectionHandler connectionHandler;
// 根据顺序存储每个命令对应的Client
private Queue<Client> clients = new LinkedList<Client>();
private boolean hasDataInBuff = false; // 是否有数据在缓存区
/**
* 根据jedisCluster实例生成对应的JedisClusterPipeline
*/
public JedisClusterPipeLine() {
JedisCluster jedisCluster = JedisClusterUtil.getJedisClusterInstance();
//根据jedisCluster实例获取其父类BinaryJedisCluster的属性值
connectionHandler = ReflectUtil.getObject(jedisCluster,"connectionHandler");
//根据刚刚从jedisCluster实例获取的connectionHandler实例
// 获取其父类JedisClusterConnectionHandler的属性值
clusterInfoCache = ReflectUtil.getObject(connectionHandler,"cache");
}
/**
* 由于集群模式存在节点的动态添加删除,且client不能实时感知(只有在执行命令时才可能知道集群发生变更),
* 因此,该实现不保证一定成功,建议在批量操作之前调用 refreshCluster() 方法重新获取集群信息。
* 也可以定时刷新以便获取最新的集群信息
* 还有可以在捕获异常中刷新集群节点信息,重试操作,设置重试次数,不能无线重试
*/
public void refreshClusterInfo(){
connectionHandler.renewSlotCache();
}
@Override
protected Client getClient(String key) {
byte[] bKey = SafeEncoder.encode(key);
return getClient(bKey);
}
@Override
protected Client getClient(byte[] bkey) {
//1.根据key值计算槽点
Jedis jedis = getJedis(JedisClusterCRC16.getSlot(bkey));
Client client = jedis.getClient();
clients.add(client);
return client;
}
private Jedis getJedis(int slot) {
//2.同一个节点下的所有槽点获取到的JedisPool都是一样的
JedisPool slotJedisPool = clusterInfoCache.getSlotPool(slot);
//3.验证分配到同一个节点的所有槽点对应的JedisPool是否已经存在
Jedis jedis = poolToJedisMap.get(slotJedisPool);
//4. 如果不存在
if(jedis == null){
//4.1 则获取客户端连接,存储到容器中去
jedis = slotJedisPool.getResource();
poolToJedisMap.put(slotJedisPool,jedis);
}
//4.2 如果已经存储直接返回
hasDataInBuff = true;
return jedis;
}
@Override
public void close() {
clean();
//清除客户端队列
clients.clear();
//关闭缓存的jedis连接
for (Jedis jedis : poolToJedisMap.values()) {
if (hasDataInBuff) {
flushCachedData(jedis);
}
jedis.close();
}
//清除jedis连接缓存数据
poolToJedisMap.clear();
//还原初始状态
hasDataInBuff = false;
}
private void flushCachedData(Jedis jedis) {
try {
jedis.getClient().getAll();
} catch (RuntimeException ex) {
//捕获异常,保证代码的高可用
}
}
/**
* 同步读取所有数据. 与syncAndReturnAll()相比,sync()只是没有对数据做反序列化
*/
public void sync() {
innerSync(null);
}
/**
* 同步读取所有数据 并按命令顺序返回一个列表
*
* @return 按照命令的顺序返回所有的数据
*/
public List<Object> syncAndReturnAll() {
List<Object> responseList = new ArrayList<Object>();
innerSync(responseList);
return responseList;
}
private void innerSync(List<Object> responseList){
if(responseList == null){
responseList = new ArrayList<>();
}
HashSet<Client> clientSet = new HashSet<Client>();
try {
for(Client client : clients){
// 在sync()调用时其实是不需要解析结果数据的,
// 但是如果不调用get方法,发生了JedisMovedDataException这样的错误应用是不知道的,
// 因此需要调用get()来触发错误。
// 其实如果Response的data属性可以直接获取,可以省掉解析数据的时间,
// 然而它并没有提供对应方法,要获取data属性就得用反射,不想再反射了,所以就这样了
Object response = generateResponse(client.getOne()).get();
if(response != null){
responseList.add(response);
}
// size相同说明所有的client都已经添加,就不用再调用add方法了
if (clientSet.size() != poolToJedisMap.size()) {
clientSet.add(client);
}
}
} catch (JedisRedirectionException ex) {
if(ex instanceof JedisMovedDataException){
// if MOVED redirection occurred, rebuilds cluster's slot cache,
// recommended by Redis cluster specification
refreshClusterInfo();
}
} finally {
if (clientSet.size() != poolToJedisMap.size()) {
// 所有还没有执行过的client要保证执行(flush),防止放回连接池后后面的命令被污染
for (Jedis jedis : poolToJedisMap.values()) {
if (clientSet.contains(jedis.getClient())) {
continue;
}
flushCachedData(jedis);
}
}
hasDataInBuff = false;
close();
}
}
}
4.junit测试类
‘ 1.通过pipeline命令
package com.roger.redis.cluster.pipeline;
import org.junit.Before;
import org.junit.Test;
import java.util.List;
import static org.junit.Assert.*;
public class JedisClusterPipeLineTest {
private JedisClusterPipeLine jedisClusterPipeLine;
@Before
public void setUp(){
jedisClusterPipeLine = new JedisClusterPipeLine();
}
@Test
public void testJedisClusterPipeLine() {
long startTime = System.currentTimeMillis();
//获取最新的集群节点信息
jedisClusterPipeLine.refreshClusterInfo();
List<Object> batchResult = null;
try {
// batch write
for (int i = 0; i < 10000; i++) {
jedisClusterPipeLine.set("k" + i, "v1" + i);
}
jedisClusterPipeLine.sync();
// batch read
for (int i = 0; i < 10000; i++) {
jedisClusterPipeLine.get("k" + i);
}
batchResult = jedisClusterPipeLine.syncAndReturnAll();
} finally {
jedisClusterPipeLine.close();
}
long endTime = System.currentTimeMillis();
System.out.println("总共耗时:" + (endTime - startTime) + "ms; " + (endTime-startTime)/1000 + "s");
System.out.println("结果集数量" + batchResult.size());
}
}
2.不通过pipe命令
package com.roger.redis.cluster;
import org.junit.Before;
import org.junit.Test;
import redis.clients.jedis.JedisCluster;
import java.util.List;
public class JedisClusterUtilTest {
private JedisCluster jedisCluster;
@Before
public void setUp() {
jedisCluster = JedisClusterUtil.getJedisClusterInstance();
}
@Test
public void testJedisCluster() {
long startTime = System.currentTimeMillis();
List<Object> batchResult = new ArrayList<>();
// batch write
for (int i = 0; i < 10000; i++) {
jedisCluster.set("k" + i, "v1" + i);
}
// batch read
for (int i = 0; i < 10000; i++) {
batchResult.add(jedisCluster.get("k" + i));
}
long endTime = System.currentTimeMillis();
System.out.println("总共耗时:" + (endTime - startTime) + "ms; " + (endTime - startTime) / 1000 + "s");
System.out.println("结果集数量" + batchResult.size());
}
}
5.JedisClusterPipeLine实现原理
5.1) JedisCluster相关类图
5.2) getClient() 方法的原理
5.2.1) 根据key值获取到槽点
5.2.2) 根据槽点获取到JedisPool,又因为同一个节点下所有槽点都对应一个JedisPool
5.2.3) 定义一个Map<JedisPool,Jedis> poolToJedisMap;含义给批量存储的key值按照槽点分离,
因为5.2.2)获取到的JedisPool作为key存储到poolToJedisMap容器中去,如果已经存在,则不在存储,如果没有
则把5.2.2)获取到的JedisPool作为key存储起来