Sentinel集群的搭建和测试
master: 127.0.0.16379
slave1: 127.0.0.1 6380
slave2: 127.0.0.1 6381
master-sentinel: 127.0.0.1 26379
slave1-sentinel: 127.0.0.1 26380
slave2-sentinel: 127.0.0.1 26381
1.2 集群配置
1) Master
redis.conf:
daemonize yes
port 6379
requirepass pw
masterauth pw
appendonly no
save ""
slave-read-only yes
6379-sentinal.conf
port 26379
sentinel monitor mymaster 192.168.0.105 6379 2
sentinel auth-pass mymaster foobared
sentinel down-after-milliseconds mymaster 30000
sentinel parallel-syncs mymaster 1
sentinel failover-timeout mymaster 900000
2) Salve1
Redis.conf
daemonize yes
port 6380
requirepass pw
masterauth pw
appendonly no
save ""
slave-read-only yes
slaveof 192.168.0.105 6379
6380-sentinal.conf
port 26380
sentinel monitormymaster 192.168.0.105 6379 2
sentinel auth-passmymaster pw
sentineldown-after-milliseconds mymaster 30000
sentinel parallel-syncsmymaster 1
sentinel failover-timeout mymaster 900000
3) Salve2
Redis.conf
daemonize yes
port 6381
requirepass pw
masterauth pw
appendonly no
save ""
slave-read-only yes
slaveof 192.168.0.105 6379
6381-sentinal.conf
port 26381
sentinel monitormymaster 192.168.0.105 6379 2
sentinel auth-passmymaster pw
sentineldown-after-milliseconds mymaster 30000
sentinel parallel-syncsmymaster 1
sentinel failover-timeout mymaster 900000
1.3 启动集群
写一个批量启动集群的shell脚本:
St-batch-start.sh
./bin/redis-server./st_cluster_1/master-6379/redis.conf&
./bin/redis-sentinel./st_cluster_1/master-6379/6379-sentinal.conf&
./bin/redis-server./st_cluster_1/salve-6380/redis.conf&
./bin/redis-sentinel ./st_cluster_1/salve-6380/6380-sentinal.conf&
./bin/redis-server./st_cluster_1/salve-6381/redis.conf&
./bin/redis-sentinel./st_cluster_1/salve-6381/6381-sentinal.conf&
#sh st-batch-start.sh
写一个批量关闭集群的shell脚本:
St-batch-stop.sh
kill -9 `ps -ef | grep redis | awk '{print $2}'`
#sh st-batch-stop.sh
1.1 启动后的结果
起一个会话查看master的情况:
redis-cli -h 192.168.0.105 -p 6379
auth pw
info Replication
起一个会话查询salve1的情况:
redis-cli -h192.168.0.105 -p 6380
起一个会话查询salve2的情况:
redis-cli -h192.168.0.105 -p 6381
此时查看sentinal各个主机的配置文件:
端口为26379的sentinel的配置文件没有改变:
端口为26380的sentinel的配置文件有改变:
端口为26381的文件被重写了,因为它们发现了master的salve 6381,和另一个sentinel 26381。
1.4 场景测试
----场景1:master宕机
master-sentinel作为master 1的leader,会选取一个master 1的slave作为新的master。slave的选取是根据一个判断DNS情况的优先级来得到,优先级相同通过runid的排序得到,但目前优先级设定还没实现,所以直接获取runid排序得到slave 1。然后发送命令slaveofno one来取消slave 1的slave状态来转换为master。当其他sentinel观察到该slave成为master后,就知道错误处理例程启动了。sentinel A然后发送给其他slave slaveof new-slave-ip-port 命令,当所有slave都配置完后,sentinelA从监测的masters列表中删除故障master,然后通知其他sentinels。
关闭master:
现在6379是master,6380和6381是salve。
Sentinel底下观察选举新的master的过程:
显示了failover的过程:
选择6380为master:
----场景2:master恢复
重新启动原来的master:
./bin/redis-server./st_cluster_1/master-6379/redis.conf&
查看sentinel状态:
原来的master自动切换成slave,不会自动恢复成master。
----场景3:salve1宕机
接着上面的继续实验。
关闭slave1:6379:
查看master的Replication信息:
此时只存在一个slave。
----场景4:salve1重启
./bin/redis-server./st_cluster_1/master-6379/redis.conf&
查看sentinel状态:
sentinel能快速的发现slave加入到集群中:
查看master的Replication信息:
1.5 Java客户端测试
package redis.clients.jedis.tests;
import java.util.HashSet;
import java.util.Set;
import org.apache.commons.pool2.impl.GenericObjectPoolConfig;
import org.junit.Before;
import org.junit.Test;
import redis.clients.jedis.HostAndPort;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisSentinelPool;
import redis.clients.jedis.Transaction;
import redis.clients.jedis.exceptions.JedisConnectionException;
import redis.clients.jedis.exceptions.JedisException;
import redis.clients.jedis.tests.utils.JedisSentinelTestUtil;
public class JedisSentinelPoolTest extends JedisTestBase {
private static final String MASTER_NAME = "mymaster";
private static final String localHost = "192.168.0.106";
protected static HostAndPort master = new HostAndPort(localHost, 6379);
protected static HostAndPort slave1 = new HostAndPort(localHost, 6380);
protected static HostAndPort slave2 = new HostAndPort(localHost, 6381);
protected static HostAndPort sentinel1 = new HostAndPort(localHost, 26379);
protected static HostAndPort sentinel2 = new HostAndPort(localHost, 26380);
protected static HostAndPort sentinel3 = new HostAndPort(localHost, 26381);
protected static Jedis sentinelJedis1;
protected static Jedis sentinelJedis2;
protected static Jedis sentinelJedis3;
protected Set<String> sentinels = new HashSet<String>();
@Before
public void setUp() throws Exception {
sentinels.add(sentinel1.toString());
sentinels.add(sentinel2.toString());
sentinels.add(sentinel3.toString());
sentinelJedis1 = new Jedis(sentinel1.getHost(), sentinel1.getPort());
sentinelJedis2 = new Jedis(sentinel2.getHost(), sentinel2.getPort());
sentinelJedis3 = new Jedis(sentinel3.getHost(), sentinel3.getPort());
}
@Test(expected = JedisConnectionException.class)
public void initializeWithNotAvailableSentinelsShouldThrowException() {
Set<String> wrongSentinels = new HashSet<String>();
wrongSentinels.add(new HostAndPort(localHost, 65432).toString());
wrongSentinels.add(new HostAndPort(localHost, 65431).toString());
JedisSentinelPool pool = new JedisSentinelPool(MASTER_NAME, wrongSentinels);
pool.destroy();
}
@Test(expected = JedisException.class)
public void initializeWithNotMonitoredMasterNameShouldThrowException() {
final String wrongMasterName = "wrongMasterName";
JedisSentinelPool pool = new JedisSentinelPool(wrongMasterName, sentinels);
pool.destroy();
}
@Test
public void checkCloseableConnections() throws Exception {
GenericObjectPoolConfig config = new GenericObjectPoolConfig();
JedisSentinelPool pool = new JedisSentinelPool(MASTER_NAME, sentinels, config, 10000,
"pw", 3);
Jedis jedis = pool.getResource();
jedis.auth("pw");
jedis.set("foo", "bar");
String ss = jedis.get("test");
assertEquals("bar", jedis.get("foo"));
jedis.close();
pool.close();
assertTrue(pool.isClosed());
}
@Test
public void ensureSafeTwiceFailover() throws InterruptedException {
JedisSentinelPool pool = new JedisSentinelPool(MASTER_NAME, sentinels,
new GenericObjectPoolConfig(), 9000, "pw", 3);
forceFailover(pool);
// after failover sentinel needs a bit of time to stabilize before a new
// failover
Thread.sleep(100);
forceFailover(pool);
// you can test failover as much as possible
}
@Test
public void returnResourceShouldResetState() {
GenericObjectPoolConfig config = new GenericObjectPoolConfig();
config.setMaxTotal(1);
config.setBlockWhenExhausted(false);
JedisSentinelPool pool = new JedisSentinelPool(MASTER_NAME, sentinels, config, 1000,
"pw", 3);
Jedis jedis = pool.getResource();
Jedis jedis2 = null;
try {
jedis.set("hello", "jedis");
Transaction t = jedis.multi();
t.set("hello", "world");
jedis.close();
jedis2 = pool.getResource();
assertTrue(jedis == jedis2);
assertEquals("jedis", jedis2.get("hello"));
} catch (JedisConnectionException e) {
if (jedis2 != null) {
jedis2 = null;
}
} finally {
jedis2.close();
pool.destroy();
}
}
@Test
public void checkResourceIsCloseable() {
GenericObjectPoolConfig config = new GenericObjectPoolConfig();
config.setMaxTotal(1);
config.setBlockWhenExhausted(false);
JedisSentinelPool pool = new JedisSentinelPool(MASTER_NAME, sentinels, config, 1000,
"pw", 3);
Jedis jedis = pool.getResource();
try {
jedis.set("hello", "jedis");
} finally {
jedis.close();
}
Jedis jedis2 = pool.getResource();
try {
assertEquals(jedis, jedis2);
} finally {
jedis2.close();
}
}
private void forceFailover(JedisSentinelPool pool) throws InterruptedException {
HostAndPort oldMaster = pool.getCurrentHostMaster();
// jedis connection should be master
Jedis beforeFailoverJedis = pool.getResource();
assertEquals("PONG", beforeFailoverJedis.ping());
waitForFailover(pool, oldMaster);
Jedis afterFailoverJedis = pool.getResource();
assertEquals("PONG", afterFailoverJedis.ping());
assertEquals("pw", afterFailoverJedis.configGet("requirepass").get(1));
assertEquals(3, afterFailoverJedis.getDB());
// returning both connections to the pool should not throw
beforeFailoverJedis.close();
afterFailoverJedis.close();
}
private void waitForFailover(JedisSentinelPool pool, HostAndPort oldMaster)
throws InterruptedException {
HostAndPort newMaster = JedisSentinelTestUtil.waitForNewPromotedMaster(MASTER_NAME,
sentinelJedis1, sentinelJedis2);
waitForJedisSentinelPoolRecognizeNewMaster(pool, newMaster);
}
private void waitForJedisSentinelPoolRecognizeNewMaster(JedisSentinelPool pool,
HostAndPort newMaster) throws InterruptedException {
while (true) {
HostAndPort currentHostMaster = pool.getCurrentHostMaster();
if (newMaster.equals(currentHostMaster)) break;
System.out.println("JedisSentinelPool's master is not yet changed, sleep...");
Thread.sleep(100);
}
}
}