客户端实现主从读写分离
在搭建spring boot redis 哨兵模式的时候，发现它只是实现了，服务选举是的主服务切换而已，并没有实现所谓的客户端读写分离。
所以决定自己实现一个。
RedisTemplate获取连接是由RedisConnectionFactory来获取连接，所以直接重写里面的相关逻辑就可以了。
直接贴代码吧：
自定义CustomJedisSentinelConnectionFactory：
public class CustomJedisSentinelConnectionFactory extends JedisConnectionFactory {
/**
* redis pool proxy
*/
private JedisSentinelPoolProxy proxy;
public CustomJedisSentinelConnectionFactory(RedisSentinelConfiguration sentinelConfig, JedisPoolConfig poolConfig) {
super(sentinelConfig, poolConfig);
}
@Override
public RedisConnection getConnection() {
// 如果使用Sentinel
if (isRedisSentinelAware()) {
return proxy.getConnection();
}
return super.getConnection();
}
private int getReadTimeout() {
return Math.toIntExact(getClientConfiguration().getReadTimeout().toMillis());
}
@Override
public void afterPropertiesSet() {
// 如果使用Sentinel
if (getUsePool() && isRedisSentinelAware()) {
RedisSentinelConfiguration sentinelConfiguration = getSentinelConfiguration();
this.proxy = createJedisSentinelPoolProxy(sentinelConfiguration);
} else {
super.afterPropertiesSet();
}
}
private Set<String> convertToJedisSentinelSet(Collection<RedisNode> nodes) {
if (CollectionUtils.isEmpty(nodes)) {
return Collections.emptySet();
}
Set<String> convertedNodes = new LinkedHashSet<>(nodes.size());
for (RedisNode node : nodes) {
if (node != null) {
convertedNodes.add(node.asString());
}
}
return convertedNodes;
}
private int getConnectTimeout() {
return Math.toIntExact(getClientConfiguration().getConnectTimeout().toMillis());
}
protected JedisSentinelPoolProxy createJedisSentinelPoolProxy(RedisSentinelConfiguration config) {
GenericObjectPoolConfig poolConfig = getPoolConfig() != null ? getPoolConfig() : new JedisPoolConfig();
return new JedisSentinelPoolProxy(Objects.requireNonNull(config.getMaster()).getName(), convertToJedisSentinelSet(config.getSentinels()),
poolConfig, getConnectTimeout(), getReadTimeout(), getPassword(), getDatabase(), getClientName());
}
}
主从服务的连接池自定义了一个JedisSentinelPoolProxy来获取连接：
public class JedisSentinelPoolProxy implements Closeable {
protected final List<MasterListener> masterListeners = new ArrayList<>();
private final Logger log = LoggerFactory.getLogger(JedisSentinelPoolProxy.class);
private final ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
private final Lock readLock = readWriteLock.readLock();
private final Lock writeLock = readWriteLock.writeLock();
protected GenericObjectPoolConfig poolConfig;
protected String masterName;
protected int connectionTimeout;
protected int soTimeout;
protected String password;
protected int database;
protected String clientName;
protected int sentinelConnectionTimeout;
protected int sentinelSoTimeout;
protected String sentinelUser;
protected String sentinelPassword;
protected String sentinelClientName;
/**
* 主库pool代理
*/
private volatile Pool<Jedis> masterPoolDelegate;
/**
* 从库代理pools
*/
private volatile List<Pool<Jedis>> slavePoolDelegates = new ArrayList<>();
/**
* 保存一下HostAndPort和pool的关系数据，用于判断
*/
private volatile Map<HostAndPort, Pool<Jedis>> hostAndPortPoolMap;
/**
* 从库负载
*/
private SlaveLoadBalanceAlgorithm slaveLoadBalanceAlgorithm = new RoundSlaveLoadBalanceAlgorithm();
public JedisSentinelPoolProxy(String masterName, Set<String> sentinels, GenericObjectPoolConfig poolConfig, int timeout, int soTimeout, String password, int database) {
this(masterName, sentinels, poolConfig, timeout, soTimeout, password, database, null);
}
public JedisSentinelPoolProxy(String masterName, Set<String> sentinels,
final GenericObjectPoolConfig poolConfig, final int connectionTimeout, final int soTimeout,
final String password, final int database, final String clientName,
final int sentinelConnectionTimeout, final int sentinelSoTimeout, final String sentinelUser,
final String sentinelPassword, final String sentinelClientName) {
this.poolConfig = poolConfig;
this.connectionTimeout = connectionTimeout;
this.soTimeout = soTimeout;
this.password = password;
this.database = database;
this.clientName = clientName;
this.sentinelConnectionTimeout = sentinelConnectionTimeout;
this.sentinelSoTimeout = sentinelSoTimeout;
this.sentinelUser = sentinelUser;
this.sentinelPassword = sentinelPassword;
this.sentinelClientName = sentinelClientName;
this.masterName = masterName;
this.hostAndPortPoolMap = new ConcurrentHashMap<>();
initSentinels(sentinels);
}
public JedisSentinelPoolProxy(String masterName, Set<String> sentinels, GenericObjectPoolConfig poolConfig, int connectionTimeout, int soTimeout, String password, int database, String clientName) {
this.poolConfig = poolConfig;
this.connectionTimeout = connectionTimeout;
this.soTimeout = soTimeout;
this.password = password;
this.database = database;
this.clientName = clientName;
this.masterName = masterName;
this.hostAndPortPoolMap = new ConcurrentHashMap<>();
this.initSentinels(sentinels);
}
private void initSentinels(Set<String> sentinels) {
HostAndPort master = null;
List<HostAndPort> slaves = new ArrayList<>();
HostAndPort hap;
Jedis jedis = null;
for (String sentinel : sentinels) {
hap = HostAndPort.parseString(sentinel);
log.info("开始尝试连接到 Sentinel " + hap + " 进行 redis 初始化....");
try {
jedis = new Jedis(hap.getHost(), hap.getPort());
//链接主服务
master = getMasterInfo(jedis);
log.info("初始化redis主服务为：{}", master);
//链接从服务
slaves = getSlaveInfos(jedis);
log.info("初始化到redis从服务为：{}", slaves);
if (master != null && slaves.isEmpty()) {
slaves.add(master);
log.info("无有效的从服务，初始化到redis从服务为主服务：{}", master);
}
break;
} catch (JedisException var13) {
log.info("无法连接到sentinel " + hap + ". 原因: " + var13 + ". 尝试下一个...");
} finally {
if (jedis != null) {
jedis.close();
}
}
}
if (master == null) {
throw new JedisException("无法初始化redis sentinel 因为找不到name为： " + masterName + " 的redis master 服务...");
} else {
for (String sentinel : sentinels) {
log.info("开始监听sentinel：{} ", sentinel);
hap = HostAndPort.parseString(sentinel);
JedisSentinelPoolProxy.MasterListener masterListener = new JedisSentinelPoolProxy.MasterListener(masterName, hap.getHost(), hap.getPort());
masterListener.setDaemon(true);
masterListeners.add(masterListener);
masterListener.start();
}
log.info("开始尝试初始化redis master pool and slave pools....");
initPools(master, slaves);
log.info("开始尝试初始化redis master pool and slave pools 成功....");
}
}
/**
* 获取sentinel中的有效slave服务
*
* @param jedis
* @return
*/
private List<HostAndPort> getSlaveInfos(Jedis jedis) {
List<HostAndPort> slaves = new ArrayList<>();
List<Map<String, String>> slaveMaps = jedis.sentinelSlaves(masterName);
if (slaveMaps.size() > 0) {
for (Map<String, String> slaveMap : slaveMaps) {
if (!slaveMap.get("flags").contains("s_down") && "ok".equals(slaveMap.get("master-link-status"))) {
String slaveIp = slaveMap.get("ip");
String slavePort = slaveMap.get("port");
slaves.add(toHostAndPort(Arrays.asList(slaveIp, slavePort)));
}
}
}
return slaves;
}
/**
* 获取sentinel中的有效master服务
*
* @param jedis
* @return
*/
private HostAndPort getMasterInfo(Jedis jedis) {
HostAndPort master = null;
List<String> masterAddress = jedis.sentinelGetMasterAddrByName(masterName);
if (masterAddress != null && masterAddress.size() == 2) {
master = this.toHostAndPort(masterAddress);
}
return master;
}
private void initPools(HostAndPort master, List<HostAndPort> slaves) {
writeLock.lock();
try {
if (this.masterPoolDelegate != null) {
this.masterPoolDelegate.destroy();
}
if (this.slavePoolDelegates != null) {
this.slavePoolDelegates.forEach(Pool::destroy);
}
hostAndPortPoolMap.clear();
this.masterPoolDelegate = new JedisPool(poolConfig, master.getHost(), master.getPort(), connectionTimeout, password);
hostAndPortPoolMap.put(master, this.masterPoolDelegate);
for (HostAndPort slave : slaves) {
JedisPool slavePool = new JedisPool(poolConfig, slave.getHost(), slave.getPort(), connectionTimeout, password);
this.slavePoolDelegates.add(slavePool);
hostAndPortPoolMap.put(slave, slavePool);
}
} finally {
writeLock.unlock();
}
}
/**
* 检查和已存在的是否一样
*
* @param master
* @param slaves
* @return true 一样 false 不一样
*/
private boolean check(HostAndPort master, List<HostAndPort> slaves) {
List<String> existHostAndPorts = hostAndPortPoolMap.keySet().stream().map(Objects::toString).collect(Collectors.toList());
log.info("是否相同检查，原数据：{}，当前数据,master:{}，slaves:{}", existHostAndPorts
, master.toString(), slaves.stream().map(Objects::toString).collect(Collectors.toList()));
if (!existHostAndPorts.contains(master.toString())) {
return false;
}
if (slaves.size() != this.slavePoolDelegates.size()) {
return false;
}
for (HostAndPort slave : slaves) {
if (!existHostAndPorts.contains(slave.toString())) {
return false;
}
}
return true;
}
private String hostAndPortToString(HostAndPort hostAndPort) {
if (hostAndPort == null) {
return Strings.EMPTY;
}
return hostAndPort.getHost() + ":" + hostAndPort.getPort();
}
private HostAndPort toHostAndPort(List<String> getMasterAddrByNameResult) {
String host = getMasterAddrByNameResult.get(0);
int port = Integer.parseInt(getMasterAddrByNameResult.get(1));
return new HostAndPort(host, port);
}
public JedisConnection getConnection() {
readLock.lock();
try {
Jedis jedis;
Pool<Jedis> pool;
if (CustomSentinelContext.isMaster()) {
pool = this.masterPoolDelegate;
} else {
HostAndPort chose = slaveLoadBalanceAlgorithm.chose(getSlaveHostAndPorts());
pool = hostAndPortPoolMap.get(chose);
}
jedis = pool.getResource();
JedisConnection jedisConnection = new JedisConnection(jedis, pool, database);
jedisConnection.setConvertPipelineAndTxResults(true);
return jedisConnection;
} finally {
readLock.unlock();
}
}
private List<HostAndPort> getSlaveHostAndPorts() {
List<HostAndPort> slaveHostAndPostList = new ArrayList<>();
for (Map.Entry<HostAndPort, Pool<Jedis>> entry : hostAndPortPoolMap.entrySet()) {
for (Pool<Jedis> slavePoolDelegate : slavePoolDelegates) {
if (entry.getValue() == slavePoolDelegate) {
slaveHostAndPostList.add(entry.getKey());
}
}
}
return slaveHostAndPostList;
}
@Override
public void close() throws IOException {
for (MasterListener masterListener : this.masterListeners) {
masterListener.shutdown();
}
this.slavePoolDelegates.forEach(Pool::destroy);
this.masterPoolDelegate.destroy();
}
protected class MasterListener extends Thread {
protected String masterName;
protected String host;
protected int port;
protected long subscribeRetryWaitTimeMillis;
protected volatile Jedis j;
protected AtomicBoolean running;
protected MasterListener() {
this.subscribeRetryWaitTimeMillis = 5000L;
this.running = new AtomicBoolean(false);
}
public MasterListener(String masterName, String host, int port) {
this.subscribeRetryWaitTimeMillis = 5000L;
this.running = new AtomicBoolean(false);
this.masterName = masterName;
this.host = host;
this.port = port;
}
public MasterListener(String masterName, String host, int port, long subscribeRetryWaitTimeMillis) {
this(masterName, host, port);
this.subscribeRetryWaitTimeMillis = subscribeRetryWaitTimeMillis;
}
@Override
public void run() {
this.running.set(true);
while (this.running.get()) {
this.j = new Jedis(this.host, this.port);
try {
if (!this.running.get()) {
break;
}
this.j.subscribe(new JedisPubSub() {
@Override
public void onMessage(String channel, String message) {
log.info("接收到消息：{},channel:{}", message, channel);
//主服务宕机,重新设置主从
if ("+switch-master".equals(channel)) {
String[] switchMasterMsg = message.split(" ");
if (switchMasterMsg.length > 3) {
if (masterName.equals(switchMasterMsg[0])) {
log.info("sentinel发生服务选举，开始重新初始化....");
reInitMasterPool();
} else {
log.info("sentinel 发生服务选举，但是:{}与当前：{}不符，不刷新pool...", switchMasterMsg[0], masterName);
}
} else {
log.error("sentinel 发生服务选举，返回异常参数：{}", switchMasterMsg);
}
}
//实例下线了，如果是从服务下线，重新分配slave
else if ("+sdown".equals(channel)) {
String[] sdownMsg = message.split(" ");
if (CustomSentinelContext.SLAVE.equals(sdownMsg[0])) {
log.info("sentinel发生服务下线了，开始重新初始化slave....");
reInitSlavePools();
} else {
log.error("sentinel发生服务下线了，但是返回错误参数：{}", (Object) sdownMsg);
}
}
// 从节点现在和主节点同步
else if ("+slave-reconf-done".equals(channel)) {
log.info("从节点现在和主节点同步,:{},开始重新初始化slave....", message);
reInitSlavePools();
}
// 服务主观上线了
else if ("-sdown".equals(channel)) {
log.info("服务主观上线了:{},开始重新初始化slave....", message);
String[] sdownMsg = message.split(" ");
if (CustomSentinelContext.SLAVE.equals(sdownMsg[0])) {
reInitSlavePools();
} else {
log.error("服务主观上线了，但是返回错误参数：{}", (Object) sdownMsg);
}
}
}
}, "+switch-master", "+sdown", "+slave-reconf-done", "-sdown", "selected-slave");
} catch (JedisConnectionException var8) {
if (this.running.get()) {
log.error("丢失sentinel连接： " + this.host + ":" + this.port + ". 睡眠5秒后重新尝试....", var8);
try {
Thread.sleep(this.subscribeRetryWaitTimeMillis);
} catch (InterruptedException var7) {
log.error("Sleep interrupted: ", var7);
}
} else {
log.error("Unsubscribing from Sentinel at " + this.host + ":" + this.port);
}
} finally {
this.j.close();
}
}
}
/**
* 重新刷新slave
*/
private void reInitSlavePools() {
Jedis jedis = new Jedis(host, port);
List<HostAndPort> slaveInfos = getSlaveInfos(jedis);
writeLock.lock();
try {
log.info("reInitSlavePools====重新查询的从服务为：{}", slaveInfos);
if (CollectionUtil.isEmpty(slaveInfos)) {
HostAndPort masterInfo = getMasterInfo(jedis);
if (masterInfo != null) {
slaveInfos.add(masterInfo);
}
}
// 检查配置是否一样
boolean isDiff = false;
if (slavePoolDelegates.size() != slaveInfos.size()) {
isDiff = true;
}
List<HostAndPort> slaveHostAndPorts = getSlaveHostAndPorts();
for (HostAndPort slaveInfo : slaveInfos) {
if (!slaveHostAndPorts.stream().map(JedisSentinelPoolProxy.this::hostAndPortToString)
.collect(Collectors.toList()).contains(hostAndPortToString(slaveInfo))) {
isDiff = true;
}
}
log.info("reInitSlavePools====当前存在配置：{}，需要变更的配置：{}", slaveHostAndPorts, slaveInfos);
if (isDiff) {
slaveHostAndPorts.forEach(hostAndPortPoolMap::remove);
slavePoolDelegates.clear();
for (HostAndPort slave : slaveInfos) {
JedisPool slavePool = new JedisPool(poolConfig, slave.getHost(), slave.getPort(), connectionTimeout, password);
slavePoolDelegates.add(slavePool);
hostAndPortPoolMap.put(slave, slavePool);
}
log.info("reInitSlavePools====重置配置为：{}", slaveInfos);
} else {
log.info("reInitSlavePools====不需要重新配置");
}
} finally {
writeLock.unlock();
}
}
/**
* 重新刷新主服务
*/
private void reInitMasterPool() {
Jedis jedis = new Jedis(this.host, this.port);
HostAndPort newMaster = getMasterInfo(jedis);
writeLock.lock();
try {
HostAndPort master = null;
for (Map.Entry<HostAndPort, Pool<Jedis>> entry : hostAndPortPoolMap.entrySet()) {
if (entry.getValue() == masterPoolDelegate) {
master = entry.getKey();
}
}
if (master != null) {
log.info("reInitMasterPool====当前配置：{}，变更配置：{}", hostAndPortToString(master), hostAndPortToString(newMaster));
if (!hostAndPortToString(master).equals(hostAndPortToString(newMaster))) {
hostAndPortPoolMap.remove(master);
masterPoolDelegate = new JedisPool(poolConfig, master.getHost(), master.getPort(), connectionTimeout, password);
hostAndPortPoolMap.put(newMaster, masterPoolDelegate);
log.info("reInitMasterPool====重置master为：{}", hostAndPortToString(newMaster));
} else {
log.info("reInitMasterPool====不需要变更配置");
}
}
} finally {
writeLock.unlock();
}
}
public void shutdown() {
try {
log.error("Shutting down listener on " + this.host + ":" + this.port);
this.running.set(false);
if (this.j != null) {
this.j.disconnect();
}
} catch (Exception var2) {
log.error("Caught exception while shutting down: ", var2);
}
}
}
}
从服务负载均衡策略类：
public interface SlaveLoadBalanceAlgorithm {
HostAndPort chose(List<HostAndPort> slaveHostAndPosts);
}
public class RoundSlaveLoadBalanceAlgorithm implements SlaveLoadBalanceAlgorithm {
private final AtomicInteger index = new AtomicInteger(0);
@Override
public HostAndPort chose(List<HostAndPort> slaveHostAndPosts) {
int i = index.getAndIncrement();
return slaveHostAndPosts.get(i % slaveHostAndPosts.size());
}
}
相关辅助类：
public class CustomSentinelContext {
/**
* 主类型
*/
public static final String MASTER = "master";
/**
* 从类型
*/
public static final String SLAVE = "slave";
/**
* 当前上下文
*/
public static ThreadLocal<String> sentinelCurrentCxt = ThreadLocal.withInitial(() -> MASTER);
/**
* 主库都
*/
public static void master() {
sentinelCurrentCxt.set(MASTER);
}
/**
* 从库读
*/
public static void slave() {
sentinelCurrentCxt.set(SLAVE);
}
public static boolean isMaster() {
return sentinelCurrentCxt.get().equals(MASTER);
}
}
public class CustomRedisTemplate extends RedisTemplate<String, Object> {
public CustomRedisTemplate master() {
CustomSentinelContext.master();
return this;
}
public CustomRedisTemplate slave() {
CustomSentinelContext.slave();
return this;
}
}
配置类：
@Configuration
@AutoConfigureAfter(value = RedisConfig.class)
public class RedisSentinelConfig {
private final RedisProperties redisProperties;
public RedisSentinelConfig(RedisProperties redisProperties) {
this.redisProperties = redisProperties;
}
@Bean(value = "customRedisTemplate")
@Primary
public CustomRedisTemplate customRedisTemplate(RedisSerializer<Object> redisSerializer) {
CustomRedisTemplate redisTemplate = new CustomRedisTemplate();
RedisKeySerializer redisKeySerializer = new RedisKeySerializer();
// key 序列化
redisTemplate.setKeySerializer(redisKeySerializer);
redisTemplate.setHashKeySerializer(redisKeySerializer);
// value 序列化
redisTemplate.setValueSerializer(redisSerializer);
redisTemplate.setHashValueSerializer(redisSerializer);
redisTemplate.setConnectionFactory(redisSentinelConnectionFactory());
return redisTemplate;
}
@Bean(value = "redisSentinelConnectionFactory")
public RedisConnectionFactory redisSentinelConnectionFactory() {
RedisProperties.Sentinel sentinel = redisProperties.getSentinel();
RedisSentinelConfiguration redisSentinelConfiguration = new RedisSentinelConfiguration(sentinel.getMaster(), new HashSet<>(sentinel.getNodes()));
redisSentinelConfiguration.setDatabase(redisProperties.getDatabase());
redisSentinelConfiguration.setPassword(redisProperties.getPassword());
redisSentinelConfiguration.setSentinelPassword(sentinel.getPassword());
return new CustomJedisSentinelConnectionFactory(redisSentinelConfiguration, new JedisPoolConfig());
}
}
配置
spring.redis.database=0
spring.redis.host=172.30.60.128
spring.redis.password=123456
spring.redis.port=7000
spring.redis.sentinel.master=mymaster
spring.redis.sentinel.nodes=172.30.60.128:27001,172.30.60.128:27002,172.30.60.128:27003
xiao7.redis.enable=true
server.port=8081
5696
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
