zookeeper的python客户端

最新推荐文章于 2021-02-24 23:07:53 发布

tycoon1988

最新推荐文章于 2021-02-24 23:07:53 发布

阅读量1.3k

点赞数

本文链接：https://blog.csdn.net/tycoon1988/article/details/38444645

版权

http://blog.csdn.net/lengzijian/article/details/9233327

一篇已经讲解了如何安装zookeeper的python客户端，接下来是我在网上搜到的例子，举例应用环境是：

1.当有两个或者多个服务运行，并且同意时间只有一个服务接受请求(工作)，其他服务待命。

2.当接受请求(工作)的服务异常挂掉时，会从剩下的待命服务中选举出一个服务来接受请求(工作)。

下面直接上例子，有两个文件组成1.zkclient.py 2.zktest.py

# coding: utf-8
# modfied from https://github.com/phunt/zk-smoketest/blob/master/zkclient.py
# zkclient.py

import zookeeper, time, threading
from collections import namedtuple

DEFAULT_TIMEOUT = 30000
VERBOSE = True

ZOO_OPEN_ACL_UNSAFE = {"perms":0x1f, "scheme":"world", "id" :"anyone"}

# Mapping of connection state values to human strings.
STATE_NAME_MAPPING = {
    zookeeper.ASSOCIATING_STATE: "associating",
    zookeeper.AUTH_FAILED_STATE: "auth-failed",
    zookeeper.CONNECTED_STATE: "connected",
    zookeeper.CONNECTING_STATE: "connecting",
    zookeeper.EXPIRED_SESSION_STATE: "expired",
}

# Mapping of event type to human string.
TYPE_NAME_MAPPING = {
    zookeeper.NOTWATCHING_EVENT: "not-watching",
    zookeeper.SESSION_EVENT: "session",
    zookeeper.CREATED_EVENT: "created",
    zookeeper.DELETED_EVENT: "deleted",
    zookeeper.CHANGED_EVENT: "changed",
    zookeeper.CHILD_EVENT: "child",
}

class ZKClientError(Exception):
    def __init__(self, value):
        self.value = value
    def __str__(self):
        return repr(self.value)

class ClientEvent(namedtuple("ClientEvent", 'type, connection_state, path')):
    """
    A client event is returned when a watch deferred fires. It denotes
    some event on the zookeeper client that the watch was requested on.
    """

    @property
    def type_name(self):
        return TYPE_NAME_MAPPING[self.type]

    @property
    def state_name(self):
        return STATE_NAME_MAPPING[self.connection_state]

    def __repr__(self):
        return "<ClientEvent %s at %r state: %s>" % (
            self.type_name, self.path, self.state_name)

def watchmethod(func):
    def decorated(handle, atype, state, path):
        event = ClientEvent(atype, state, path)
        return func(event)
    return decorated

class ZKClient(object):
    def __init__(self, servers, timeout=DEFAULT_TIMEOUT):
        self.timeout = timeout
        self.connected = False
        self.conn_cv = threading.Condition( )
        self.handle = -1

        self.conn_cv.acquire()
        if VERBOSE: print("Connecting to %s" % (servers))
        start = time.time()
        self.handle = zookeeper.init(servers, self.connection_watcher, timeout)
        self.conn_cv.wait(timeout/1000)
        self.conn_cv.release()

if not self.connected:
raise ZKClientError("Unable to connect to %s" % (servers))

        if VERBOSE:
            print("Connected in %d ms, handle is %d"
                  % (int((time.time() - start) * 1000), self.handle))

    def connection_watcher(self, h, type, state, path):
        self.handle = h
        self.conn_cv.acquire()
        self.connected = True
        self.conn_cv.notifyAll()
        self.conn_cv.release()

    def close(self):
        return zookeeper.close(self.handle)

    def create(self, path, data="", flags=0, acl=[ZOO_OPEN_ACL_UNSAFE]):
        start = time.time()
        result = zookeeper.create(self.handle, path, data, acl, flags)
        if VERBOSE:
            print("Node %s created in %d ms"
                  % (path, int((time.time() - start) * 1000)))
        return result

    def delete(self, path, version=-1):
        start = time.time()
        result = zookeeper.delete(self.handle, path, version)
        if VERBOSE:
            print("Node %s deleted in %d ms"
                  % (path, int((time.time() - start) * 1000)))
        return result

def get(self, path, watcher=None):
return zookeeper.get(self.handle, path, watcher)

def exists(self, path, watcher=None):
return zookeeper.exists(self.handle, path, watcher)

def set(self, path, data="", version=-1):
return zookeeper.set(self.handle, path, data, version)

def set2(self, path, data="", version=-1):
return zookeeper.set2(self.handle, path, data, version)

def get_children(self, path, watcher=None):
return zookeeper.get_children(self.handle, path, watcher)

def async(self, path = "/"):
return zookeeper.async(self.handle, path)

    def acreate(self, path, callback, data="", flags=0, acl=[ZOO_OPEN_ACL_UNSAFE]):
        result = zookeeper.acreate(self.handle, path, data, acl, flags, callback)
        return result

def adelete(self, path, callback, version=-1):
return zookeeper.adelete(self.handle, path, version, callback)

def aget(self, path, callback, watcher=None):
return zookeeper.aget(self.handle, path, watcher, callback)

def aexists(self, path, callback, watcher=None):
return zookeeper.aexists(self.handle, path, watcher, callback)

def aset(self, path, callback, data="", version=-1):
return zookeeper.aset(self.handle, path, data, version, callback)

watch_count = 0

"""Callable watcher that counts the number of notifications"""
class CountingWatcher(object):
    def __init__(self):
        self.count = 0
        global watch_count
        self.id = watch_count
        watch_count += 1

    def waitForExpected(self, count, maxwait):
        """Wait up to maxwait for the specified count,
        return the count whether or not maxwait reached.

        Arguments:
        - `count`: expected count
        - `maxwait`: max milliseconds to wait
        """
        waited = 0
        while (waited < maxwait):
            if self.count >= count:
                return self.count
            time.sleep(1.0);
            waited += 1000
        return self.count

    def __call__(self, handle, typ, state, path):
        self.count += 1
        if VERBOSE:
            print("handle %d got watch for %s in watcher %d, count %d" %
                  (handle, path, self.id, self.count))

"""Callable watcher that counts the number of notifications
and verifies that the paths are sequential"""
class SequentialCountingWatcher(CountingWatcher):
    def __init__(self, child_path):
        CountingWatcher.__init__(self)
        self.child_path = child_path

    def __call__(self, handle, typ, state, path):
        if not self.child_path(self.count) == path:
            raise ZKClientError("handle %d invalid path order %s" % (handle, path))
        CountingWatcher.__call__(self, handle, typ, state, path)

class Callback(object):
    def __init__(self):
        self.cv = threading.Condition()
        self.callback_flag = False
        self.rc = -1

    def callback(self, handle, rc, handler):
        self.cv.acquire()
        self.callback_flag = True
        self.handle = handle
        self.rc = rc
        handler()
        self.cv.notify()
        self.cv.release()

    def waitForSuccess(self):
        while not self.callback_flag:
            self.cv.wait()
        self.cv.release()

        if not self.callback_flag == True:
            raise ZKClientError("asynchronous operation timed out on handle %d" %
                             (self.handle))
        if not self.rc == zookeeper.OK:
            raise ZKClientError(
                "asynchronous operation failed on handle %d with rc %d" %
                (self.handle, self.rc))

class GetCallback(Callback):
def __init__(self):
Callback.__init__(self)

    def __call__(self, handle, rc, value, stat):
        def handler():
            self.value = value
            self.stat = stat
        self.callback(handle, rc, handler)

class SetCallback(Callback):
def __init__(self):
Callback.__init__(self)

    def __call__(self, handle, rc, stat):
        def handler():
            self.stat = stat
        self.callback(handle, rc, handler)

class ExistsCallback(SetCallback):
pass

class CreateCallback(Callback):
def __init__(self):
Callback.__init__(self)

    def __call__(self, handle, rc, path):
        def handler():
            self.path = path
        self.callback(handle, rc, handler)

class DeleteCallback(Callback):
def __init__(self):
Callback.__init__(self)

    def __call__(self, handle, rc):
        def handler():
            pass
        self.callback(handle, rc, handler)

http://www.cnblogs.com/lexus/archive/2012/03/11/2390203.html

上面的文件是别人封装好的zookeeper接口，下面是测试代码，需要依赖上面的包：

# coding: utf-8
# zktest.py

import logging
from os.path import basename, join

from zkclient import ZKClient, zookeeper, watchmethod

logging.basicConfig(
level = logging.DEBUG,
format = "[%(asctime)s] %(levelname)-8s %(message)s"
)

log = logging

class GJZookeeper(object):

    ZK_HOST = "localhost:2181"
    ROOT = "/app"
    WORKERS_PATH = join(ROOT, "workers")
    MASTERS_NUM = 1
    TIMEOUT = 10000

    def __init__(self, verbose = True):
        self.VERBOSE = verbose
        self.masters = []
        self.is_master = False
        self.path = None

        self.zk = ZKClient(self.ZK_HOST, timeout = self.TIMEOUT)
        self.say("login ok!")
        # init
        self.__init_zk()
        # register
        self.register()

    def __init_zk(self):
        """
        create the zookeeper node if not exist
        """
        nodes = (self.ROOT, self.WORKERS_PATH)
        for node in nodes:
            if not self.zk.exists(node):
                try:
                    self.zk.create(node, "")
                except:
                    pass

    @property
    def is_slave(self):
        return not self.is_master

    def register(self):
        """
        register a node for this worker
        """
        self.path = self.zk.create(self.WORKERS_PATH + "/worker", "1", flags=zookeeper.EPHEMERAL | zookeeper.SEQUENCE)
        self.path = basename(self.path)
        self.say("register ok! I'm %s" % self.path)
        # check who is the master
        self.get_master()

    def get_master(self):
        """
        get children, and check who is the smallest child
        """
        @watchmethod
        def watcher(event):
            self.say("child changed, try to get master again.")
            self.get_master()

        children = self.zk.get_children(self.WORKERS_PATH, watcher)
        children.sort()
        self.say("%s's children: %s" % (self.WORKERS_PATH, children))

        # check if I'm master
        self.masters = children[:self.MASTERS_NUM]
        if self.path in self.masters:
            self.is_master = True
            self.say("I've become master!")
        else:
            self.say("%s is masters, I'm slave" % self.masters)

    def say(self, msg):
        """
        print messages to screen
        """
        if self.VERBOSE:
            if self.path:
                if self.is_master:
                    log.info("[ %s(%s) ] %s" % (self.path, "master" , msg))
                else:
                    log.info("[ %s(%s) ] %s" % (self.path, "slave", msg))
            else:
                log.info(msg)

def main():
gj_zookeeper = GJZookeeper()

if __name__ == "__main__":
    main()
    import time
    time.sleep(1000)

这个简单的demo所做的事情，就是通过在zookeeper的/app/workers节点下建立临时的子节点( flags=zookeeper.EPHEMERAL | zookeeper.SEQUENCE )，每次create完成之后检查自己是不是在最小的MASTERS_NUM(例子中为1，即单master)里。如果是的话，作为master运行，否则的话，作为slave运行。

这样的话，当我们的master挂掉以后，与zookeeper之间的连接也会中断，过了指定的TIMEOUT以后，master之前在worker下的子节点就会被删除，于是slave节点之前设置的watcher会被触发，再次检查自己是否为master，如果是的话则完成切换。

demo运行结果：

 
        # 第一个实例 
       
        Connected in 20 ms, handle is 0 
       
        [2011-09-09 12:40:43,702] INFO     login ok! 
       
        Node /app/workers/worker created in 4 ms 
       
        [2011-09-09 12:40:43,708] INFO     [ worker0000000022(slave) ] register ok! I'm worker0000000022 
       
        [2011-09-09 12:40:43,709] INFO     [ worker0000000022(slave) ] /app/workers's children: ['worker0000000022'] 
       
        [2011-09-09 12:40:43,709] INFO     [ worker0000000022(master) ] I've become master! 
       
        # 这时再起第二个实例 
       
        Connected in 64 ms, handle is 0 
       
        [2011-09-09 12:43:08,334] INFO     login ok! 
       
        Node /app/workers/worker created in 11 ms 
       
        [2011-09-09 12:43:08,346] INFO     [ worker0000000023(slave) ] register ok! I'm worker0000000023 
       
        [2011-09-09 12:43:08,347] INFO     [ worker0000000023(slave) ] /app/workers's children: ['worker0000000022', 'worker0000000023'] 
       
        [2011-09-09 12:43:08,347] INFO     [ worker0000000023(slave) ] ['worker0000000022'] is masters, I'm slave 
       
        # 杀掉master，第二个实例发生的变化 
       
        [2011-09-09 12:44:06,016] INFO     [ worker0000000023(slave) ] child changed, try to get master again. 
       
        [2011-09-09 12:44:06,017] INFO     [ worker0000000023(slave) ] /app/workers's children: ['worker0000000023'] 
       
        [2011-09-09 12:44:06,017] INFO     [ worker0000000023(master) ] I've become master!