1、介绍
应该都用过jdk自带的countdownlatch、cyclicbarrier和queue,都知道前者是一个可循环使用的多线程同步栅栏,后者是一个队列,用于常用于异步操作。但在分布式环境下如何做到用barrier进行同步多台机器实例的运行?如何进行队列消费?也许想到了消息队列jmx,或者用redis等分布式nosql缓存做同步,但其实zookeeper也可以,只要好好使用好getChildren这个方法。
2、barrier
barrier同步栅栏,可以对多个机器或线程进行步调协同,类似于赛马游戏,只有所有赛道上的马都准备好了,比赛才会开始,只有赛马都过了终点才算比赛结束,barrier就是为了解决这个问题。
如上图所示,类似与countdownlatch,我们可以在指定节点(如barrier1)下,每一个线程或机器都创建一个属于自己的“临时有序节点”(如192.168.1.10000000001),然后监听节点(如barrier1)的变更通知,每次收到barrier1的变更通知(使用watch),则会调用getChildren,获取当前barrier1下的子节点个数,一旦满足要求,比如3,则视为“比赛开始”;
如何判断“比赛结束”?只要满足getChildren().size() == 0即可视为比赛结束。
3、queue
queue的案例是一个producer-consumer的案例,即producer将消息发送给队列(实际上是在/queue1节点下创建了一个有序节点),而consumer则轮询方式不断从/queue1节点调用getChildren列表,从列表中获取序号最小的节点进行消费,消费完毕即删除该节点。
基本思路和barrier类似,就是充分利用getChildren及有序节点,getChildren获取所有子节点列表(也就是队列),而有序节点则保证了队列是有序。同时,zk.delete的操作也是原子的,保证了消息只能被消费一次。
有几个缺点:
- 单个消息不能太大(kb级别),因为是用节点存储一个队列消息,zk本身不是存储系统,每个节点存储量有限
- 队列长度也有限,且队列过长,还需要从队列中获取序号最小的节点,本身也有耗时
4、代码
- 基类 SyncPrimitive
package com.xxxxx.zookeeper;
import org.apache.zookeeper.WatchedEvent;
import org.apache.zookeeper.Watcher;
import org.apache.zookeeper.ZooKeeper;
import java.io.IOException;
/**
* Created by wushiweijun on 2017/11/22.
*/
public class SyncPrimitive implements Watcher{
static ZooKeeper zk = null;
/*互斥锁*/
static Integer mutex;
String root;
public SyncPrimitive(String address) {
if(zk == null){
try {
System.out.println("Starting ZK:");
zk = new ZooKeeper(address, 3000, this);
mutex = new Integer(-1);
System.out.println("Finished starting ZK: " + zk);
} catch (IOException e) {
System.out.println(e.toString());
zk = null;
}
}
}
@Override
synchronized public void process(WatchedEvent event) {
synchronized (mutex) {
/*有变化,你醒来看下*/
mutex.notify();
}
}
}
- barrier
package com.xxxxx.zookeeper;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.ZooDefs;
import org.apache.zookeeper.data.Stat;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.List;
import java.util.Random;
/**
* Created by wushiweijun on 2017/11/22.
*/
public class Barrier extends SyncPrimitive {
private int size;
private String name;
/**
* 创建栅栏根节点,后续所有的处理应用,都是在这个节点下创建子节点
*
* @param address
* @param root
* @param size
*/
Barrier(String address, String root, int size) {
super(address);
this.root = root;
this.size = size;
// Create barrier node
if (zk != null) {
try {
/*同步调用,判断节点是否存在*/
Stat s = zk.exists(root, false);
if (s == null) {
/*创建一个znode节点*/
zk.create(root, new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE,
CreateMode.PERSISTENT);
}
} catch (KeeperException e) {
System.out
.println("Keeper exception when instantiating queue: "
+ e.toString());
} catch (InterruptedException e) {
System.out.println("Interrupted exception");
}
}
// My node name
try {
name = new String(InetAddress.getLocalHost().getCanonicalHostName().toString());
} catch (UnknownHostException e) {
System.out.println(e.toString());
}
}
/**
* Join barrier
*
* @return
* @throws KeeperException
* @throws InterruptedException
*/
boolean enter() throws KeeperException, InterruptedException{
/*创建临时有序节点*/
zk.create(root + "/" + name, new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE,
CreateMode.EPHEMERAL_SEQUENTIAL);
while (true) {
synchronized (mutex) {
List<String> list = zk.getChildren(root, true);
if (list.size() < size) {
/*没到,你睡吧*/
mutex.wait();
} else {
return true;
}
}
}
}
/**
* Wait until all reach barrier
*
* @return
* @throws KeeperException
* @throws InterruptedException
*/
boolean leave() throws KeeperException, InterruptedException{
/*删除临时有序节点*/
zk.delete(root + "/" + name, 0);
while (true) {
synchronized (mutex) {
List<String> list = zk.getChildren(root, true);
if (list.size() > 0) {
mutex.wait();
} else {
return true;
}
}
}
}
public static void main(String[] args) {
args = "localhost:2181,localhost:2182,localhost:2183 2".split(" ");
Barrier b = new Barrier(args[0], "/b1", new Integer(args[1]));
try{
boolean flag = b.enter();
System.out.println("Entered barrier: " + args[1]);
if(!flag) System.out.println("Error when entering the barrier");
} catch (KeeperException e){
} catch (InterruptedException e){
}
// Generate random integer
Random rand = new Random();
int r = rand.nextInt(100);
// Loop for rand iterations
for (int i = 0; i < r; i++) {
try {
Thread.sleep(100);
} catch (InterruptedException e) {
}
}
try{
b.leave();
} catch (KeeperException e){
} catch (InterruptedException e){
}
System.out.println("Left barrier");
}
}
- queue
package com.xxxxxx.zookeeper;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.ZooDefs;
import org.apache.zookeeper.data.Stat;
import java.nio.ByteBuffer;
import java.util.List;
/**
* 这个简单案例,producer和consumer使用的是一个整数
*
* Created by wushiweijun on 2017/11/22.
*/
public class Queue extends SyncPrimitive{
/**
* Constructor of producer-consumer queue
*
* @param address
* @param name
*/
Queue(String address, String name) {
super(address);
this.root = name;
// Create ZK node name
if (zk != null) {
try {
Stat s = zk.exists(root, false);
if (s == null) {
/*创建znode节点*/
zk.create(root, new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE,
CreateMode.PERSISTENT);
}
} catch (KeeperException e) {
System.out
.println("Keeper exception when instantiating queue: "
+ e.toString());
} catch (InterruptedException e) {
System.out.println("Interrupted exception");
}
}
}
/**
* Add element to the queue.
*
* @param i
* @return
*/
boolean produce(int i) throws KeeperException, InterruptedException{
ByteBuffer b = ByteBuffer.allocate(4);
byte[] value;
// Add child with value i
b.putInt(i);
value = b.array();
/*创建持久有序节点*/
zk.create(root + "/element", value, ZooDefs.Ids.OPEN_ACL_UNSAFE,
CreateMode.PERSISTENT_SEQUENTIAL);
return true;
}
/**
* Remove first element from the queue.
*
* @return
* @throws KeeperException
* @throws InterruptedException
*/
int consume() throws KeeperException, InterruptedException{
int retvalue = -1;
Stat stat = null;
String path = null;
// Get the first element available
while (true) {
synchronized (mutex) {
List<String> list = zk.getChildren(root, true);
if (list.size() == 0) {
System.out.println("Going to wait");
/*没到,你睡吧*/
mutex.wait();
} else {
Integer min = new Integer(list.get(0).substring(7));
for(String s : list){
/*去除掉element前缀,获取后面的序号*/
Integer tempValue = new Integer(s.substring(7));
//System.out.println("Temporary value: " + tempValue);
if(tempValue < min) min = tempValue;
}
path = String.format("/element%010d" ,min);
System.out.println("Temporary value: " + root + path);
byte[] b = zk.getData(root + path,
false, stat);
/*使用了zk的原子性,如果节点被其他消费了删除了,则这个操作将报KeeperException.NoNode*/
zk.delete(root + path, 0);
ByteBuffer buffer = ByteBuffer.wrap(b);
retvalue = buffer.getInt();
return retvalue;
}
}
}
}
public static void main(String[] args) {
// args = "localhost:2181,localhost:2182,localhost:2183 10 p".split(" ");
args = "localhost:2181,localhost:2182,localhost:2183 10 c".split(" ");
Queue q = new Queue(args[0], "/app1");
System.out.println("Input: " + args[0]);
int i;
Integer max = new Integer(args[1]);
if (args[2].equals("p")) {
System.out.println("Producer");
for (i = 0; i < max; i++)
try{
q.produce(10 + i);
} catch (KeeperException e){
} catch (InterruptedException e){
}
} else {
System.out.println("Consumer");
for (i = 0; i < max; i++) {
try{
int r = q.consume();
System.out.println("Item: " + r);
} catch (KeeperException e){
i--;
} catch (InterruptedException e){
}
}
}
}
}
5、参考
[1] http://zookeeper.apache.org/doc/trunk/zookeeperTutorial.html