zookeeper分布式队列

一、简介

在此介绍基于zookeeper实现的分布式队列。包括类似jdk队列的简单队列SimpleDistributedQueue、

分布式队列DistributedQueue、带Id的分布式队列DistributedIdQueue、带优先级的分布式队列DistributedPriorityQueue。

二、添加maven依赖

<dependency>
    <groupId>org.apache.zookeeper</groupId>
    <artifactId>zookeeper</artifactId>
    <version>3.4.6</version>
</dependency>
<dependency>
    <groupId>org.apache.curator</groupId>
    <artifactId>curator-framework</artifactId>
    <version>4.0.0</version>
</dependency>
<dependency>
    <groupId>org.apache.curator</groupId>
    <artifactId>curator-client</artifactId>
    <version>4.0.0</version>
</dependency>
<dependency>
    <groupId>org.apache.curator</groupId>
    <artifactId>curator-recipes</artifactId>
    <version>4.0.0</version>
</dependency>
三、简单队列SimpleDistributedQueue

简单队列SimpleDistributedQueue和jdk中队列类似,拥有offer()、take()方法。

示例代码:

import org.apache.commons.lang3.StringUtils;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
import org.apache.curator.framework.recipes.queue.SimpleDistributedQueue;
import org.apache.curator.retry.ExponentialBackoffRetry;

import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.function.BiConsumer;
import java.util.function.Consumer;

public class ZookeeperSimpleDistributedQueueMain {
    public static void main(String[] args) throws Exception {
        //创建zookeeper客户端
        CuratorFramework client = CuratorFrameworkFactory.newClient("127.0.0.1:2181", new ExponentialBackoffRetry(1000, 3));
        client.start();

        //指定锁路径
        String lockPath = "/zkLockRoot/lock_1";
        //创建分布式队列
        SimpleDistributedQueue simpleDistributedQueue = new SimpleDistributedQueue(client,lockPath);

        //生成线程池
        ExecutorService executor = Executors.newCachedThreadPool();
        //生产
        BiConsumer<SimpleDistributedQueue,String> putConsumer = (queue, item) -> {
            try {
                Callable<Boolean> call = () -> {
                    try {
                        TimeUnit.SECONDS.sleep((int)(Math.random()*10));
                        //向队尾中添加数据
                        queue.offer(item.getBytes());
                        System.out.println(Thread.currentThread() + "  put : "+ item);
                    } catch (Exception e) {
                    }
                    return true;
                };
                executor.submit(call);
            } catch (Exception e) {
            }
        };
        //消费
        Consumer<SimpleDistributedQueue> getConsumer = (queue) -> {
            try {
                Callable<Boolean> call = () -> {
                    try {
                        while(true){
                            //从队首取出数据
                            byte[] dataByte = queue.take();
                            String data = new String(dataByte);
                            if(StringUtils.isBlank(data)){
                                break;
                            }
                            System.out.println(Thread.currentThread() + "  get : "+ data);
                        }
                    } catch (Exception e) {
                    }
                    return true;
                };
                executor.submit(call);
            } catch (Exception e) {
            }
        };

        //分布式队列测试(5个线程生产)
        System.out.println("5个并发线程生产,测试分布式队列");
        //5个生产线程
        for (int i = 0; i < 10; i++) {
            putConsumer.accept(simpleDistributedQueue, "item"+i);
        }
        //分布式队列测试(2个线程生产)
        System.out.println("2个并发线程消息,测试分布式队列");
        //2个消费线程
        for (int i = 0; i < 2; i++) {
            getConsumer.accept(simpleDistributedQueue);
        }

        executor.shutdown();
        TimeUnit.SECONDS.sleep(20);
        client.close();
    }
}
输出:

Thread[pool-3-thread-10,5,main]  put : item9
Thread[pool-3-thread-12,5,main]  get : item9
Thread[pool-3-thread-4,5,main]  put : item3
Thread[pool-3-thread-12,5,main]  get : item3
Thread[pool-3-thread-3,5,main]  put : item2
Thread[pool-3-thread-12,5,main]  get : item2
Thread[pool-3-thread-6,5,main]  put : item5
Thread[pool-3-thread-12,5,main]  get : item5
Thread[pool-3-thread-5,5,main]  put : item4
Thread[pool-3-thread-9,5,main]  put : item8
Thread[pool-3-thread-11,5,main]  get : item4
Thread[pool-3-thread-12,5,main]  get : item8
Thread[pool-3-thread-7,5,main]  put : item6
Thread[pool-3-thread-8,5,main]  put : item7
Thread[pool-3-thread-11,5,main]  get : item6
Thread[pool-3-thread-12,5,main]  get : item7
Thread[pool-3-thread-2,5,main]  put : item1
Thread[pool-3-thread-1,5,main]  put : item0
Thread[pool-3-thread-11,5,main]  get : item1
Thread[pool-3-thread-12,5,main]  get : item0

四、分布式队列DistributedQueue

分布式队列DistributedQueue和消息队列类似,需要定义消费监听器、序列化方式。

示例代码:

import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
import org.apache.curator.framework.recipes.queue.DistributedQueue;
import org.apache.curator.framework.recipes.queue.QueueBuilder;
import org.apache.curator.framework.recipes.queue.QueueConsumer;
import org.apache.curator.framework.recipes.queue.QueueSerializer;
import org.apache.curator.framework.state.ConnectionState;
import org.apache.curator.retry.ExponentialBackoffRetry;

import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.function.BiConsumer;

public class ZookeeperDistributedQueueMain {
    public static void main(String[] args) throws Exception {
        //创建zookeeper客户端
        CuratorFramework client = CuratorFrameworkFactory.newClient("127.0.0.1:2181", new ExponentialBackoffRetry(1000, 3));
        client.start();

        //指定锁路径
        String lockPath = "/zkLockRoot/lock_1";

        //消费监听器
        QueueConsumer<String> queueConsumer = new QueueConsumer<String>() {
            //消费队列数据
            @Override
            public void consumeMessage(String message) throws Exception {
                System.out.println("message: " + message);
            }

            @Override
            public void stateChanged(CuratorFramework client, ConnectionState newState) {

            }
        };

        //序列化
        QueueSerializer<String> queueSerializer = new QueueSerializer<String>() {
            @Override
            public byte[] serialize(String item) {
                return item.getBytes();
            }

            @Override
            public String deserialize(byte[] bytes) {
                return new String(bytes);
            }
        };

        //创建分布式队列
        QueueBuilder<String> distributedBuilder = QueueBuilder.builder(client, queueConsumer, queueSerializer, lockPath);
        DistributedQueue<String> distributedQueue = distributedBuilder.buildQueue();
        distributedQueue.start();

        //生成线程池
        ExecutorService executor = Executors.newCachedThreadPool();
        //生产
        BiConsumer<DistributedQueue<String>, String> putConsumer = (queue, item) -> {
            try {
                Callable<Boolean> call = () -> {
                    try {
                        //向队尾添加数据
                        queue.put(item);
                        System.out.println(Thread.currentThread() + "  put : "+ item);
                    } catch (Exception e) {
                    }
                    return true;
                };
                executor.submit(call);
            } catch (Exception e) {
            }
        };

        //分布式队列测试(5个线程生产)
        System.out.println("5个并发线程生产,测试分布式队列");
        //5个生产线程
        for (int i = 0; i < 5; i++) {
            putConsumer.accept(distributedQueue, "item"+i);
        }

        executor.shutdown();
        TimeUnit.SECONDS.sleep(20);
        distributedQueue.close();
        client.close();
    }
}
输出:

5个并发线程生产,测试分布式队列
Thread[pool-4-thread-2,5,main]  put : item1
Thread[pool-4-thread-1,5,main]  put : item0
Thread[pool-4-thread-3,5,main]  put : item2
Thread[pool-4-thread-5,5,main]  put : item4
Thread[pool-4-thread-4,5,main]  put : item3
message: item1
message: item0
message: item2
message: item4
message: item3

五、带Id的分布式队列DistributedIdQueue

带Id的分布式队列DistributedIdQueue每条数据可以指定id,也可根据此id对数据进行删除等操作,

其它的和分布式队列DistributedQueue类似。

示例代码:

import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
import org.apache.curator.framework.recipes.queue.*;
import org.apache.curator.framework.state.ConnectionState;
import org.apache.curator.retry.ExponentialBackoffRetry;

import java.util.Random;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.function.BiConsumer;

public class ZookeeperDistributedIdQueueMain {
    public static void main(String[] args) throws Exception {
        //创建zookeeper客户端
        CuratorFramework client = CuratorFrameworkFactory.newClient("127.0.0.1:2181", new ExponentialBackoffRetry(1000, 3));
        client.start();

        //指定锁路径
        String lockPath = "/zkLockRoot/lock_1";

        //消费者
        QueueConsumer<String> queueConsumer = new QueueConsumer<String>() {
            @Override
            public void consumeMessage(String message) throws Exception {
                System.out.println("message: " + message);
            }

            @Override
            public void stateChanged(CuratorFramework client, ConnectionState newState) {

            }
        };

        //序列化
        QueueSerializer<String> queueSerializer = new QueueSerializer<String>() {
            @Override
            public byte[] serialize(String item) {
                return item.getBytes();
            }

            @Override
            public String deserialize(byte[] bytes) {
                return new String(bytes);
            }
        };

        //创建分布式队列
        QueueBuilder<String> distributedBuilder = QueueBuilder.builder(client, queueConsumer, queueSerializer, lockPath);
        DistributedIdQueue<String> distributedIdQueue = distributedBuilder.buildIdQueue();
        distributedIdQueue.start();

        //生成线程池
        ExecutorService executor = Executors.newCachedThreadPool();
        //生产
        BiConsumer<DistributedIdQueue<String>, String> putConsumer = (queue, item) -> {
            try {
                Callable<Boolean> call = () -> {
                    try {
                        //向队尾添加数据,同时指定id
                        queue.put(item, item);
                        System.out.println(Thread.currentThread() + "  put : "+ item);
                        //休眠后删除刚添加的item, 此时可能还没有被消费
                        TimeUnit.MILLISECONDS.sleep(new Random().nextInt(1000));
                        queue.remove(item);
                        System.out.println(Thread.currentThread() + "  remove : "+ item);
                    } catch (Exception e) {
                    }
                    return true;
                };
                executor.submit(call);
            } catch (Exception e) {
            }
        };

        //分布式队列测试(5个线程生产)
        System.out.println("5个并发线程生产,测试分布式队列");
        for (int i = 0; i < 5; i++) {
            putConsumer.accept(distributedIdQueue, "item"+i);
        }

        executor.shutdown();
        TimeUnit.SECONDS.sleep(3);
        distributedIdQueue.close();
        client.close();
    }
}
输出:

5个并发线程生产,测试分布式队列
Thread[pool-4-thread-2,5,main]  put : item1
Thread[pool-4-thread-5,5,main]  put : item4
Thread[pool-4-thread-4,5,main]  put : item3
Thread[pool-4-thread-3,5,main]  put : item2
Thread[pool-4-thread-1,5,main]  put : item0
message: item1
Thread[pool-4-thread-1,5,main]  remove : item0
message: item4
message: item3
message: item2
Thread[pool-4-thread-2,5,main]  remove : item1
Thread[pool-4-thread-3,5,main]  remove : item2
Thread[pool-4-thread-4,5,main]  remove : item3
Thread[pool-4-thread-5,5,main]  remove : item4

六、带优先级的分布式队列DistributedPriorityQueue

带优先级的分布式队列DistributedPriorityQueue,可指定数据优先级,优先消费值小的数据。

示例代码:

import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
import org.apache.curator.framework.recipes.queue.DistributedPriorityQueue;
import org.apache.curator.framework.recipes.queue.QueueBuilder;
import org.apache.curator.framework.recipes.queue.QueueConsumer;
import org.apache.curator.framework.recipes.queue.QueueSerializer;
import org.apache.curator.framework.state.ConnectionState;
import org.apache.curator.retry.ExponentialBackoffRetry;

import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.function.BiConsumer;

public class ZookeeperDistributedPriorityQueueMain {
    public static void main(String[] args) throws Exception {
        //创建zookeeper客户端
        CuratorFramework client = CuratorFrameworkFactory.newClient("127.0.0.1:2181", new ExponentialBackoffRetry(1000, 3));
        client.start();

        //指定锁路径
        String lockPath = "/zkLockRoot/lock_21";

        //消费者
        QueueConsumer<String> queueConsumer = new QueueConsumer<String>() {
            @Override
            public void consumeMessage(String message) throws Exception {
                System.out.println("message: " + message);
            }

            @Override
            public void stateChanged(CuratorFramework client, ConnectionState newState) {

            }
        };

        //序列化
        QueueSerializer<String> queueSerializer = new QueueSerializer<String>() {
            @Override
            public byte[] serialize(String item) {
                return item.getBytes();
            }

            @Override
            public String deserialize(byte[] bytes) {
                return new String(bytes);
            }
        };

        //创建分布式队列
        QueueBuilder<String> distributedBuilder = QueueBuilder.builder(client, queueConsumer, queueSerializer, lockPath);
        DistributedPriorityQueue<String> distributedIdQueue = distributedBuilder.buildPriorityQueue(0);
        distributedIdQueue.start();

        //生成线程池
        ExecutorService executor = Executors.newCachedThreadPool();
        //生产
        BiConsumer<DistributedPriorityQueue<String>, Integer> putConsumer = (queue, priority) -> {
            try {
                Callable<Boolean> call = () -> {
                    try {
                        queue.put("item"+priority, priority);
                        System.out.println(Thread.currentThread() + "  put : "+ "item"+priority);
                    } catch (Exception e) {
                    }
                    return true;
                };
                executor.submit(call);
            } catch (Exception e) {
            }
        };

        //分布式队列测试(5个线程生产)
        System.out.println("5个并发线程生产,测试分布式队列");
        for (int i = 0; i < 5; i++) {
            //随机指定优先级
            putConsumer.accept(distributedIdQueue, (int) (Math.random()*100));
        }

        executor.shutdown();
        TimeUnit.SECONDS.sleep(3);
        distributedIdQueue.close();
        client.close();
    }
}
输出:

5个并发线程生产,测试分布式队列
Thread[pool-4-thread-4,5,main]  put : item24
Thread[pool-4-thread-1,5,main]  put : item46
Thread[pool-4-thread-2,5,main]  put : item86
Thread[pool-4-thread-3,5,main]  put : item51
Thread[pool-4-thread-5,5,main]  put : item46
message: item24
message: item46
message: item46
message: item51
message: item86

  • 0
    点赞
  • 4
    收藏
    觉得还不错? 一键收藏
  • 0
    评论

“相关推荐”对你有帮助么?

  • 非常没帮助
  • 没帮助
  • 一般
  • 有帮助
  • 非常有帮助
提交
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值