RabbitMQ —— 四、单 Channel 消费之 ArrayBlockingQueue

RabbitMQ —— 四、单 Channel 消费之 ArrayBlockingQueue

       做过 MQ 选型的应该都清楚，RabbitMQ 的基本配置其吞吐量不如 Kafka、RocketMQ 等，这时如果又对 AMQP 比较看重，则需要尽量减少 Rabbit 服务端的计算量，比如减少其在软负载均衡方面的开销。同时在服务自动伸缩的架构中，也更容易直接观察目前的消费端实例数量。

       大体思路就是在应用内做缓冲队列，由单线程 + 单 channel 方式从 rabbit server 端接受消息放入队列，后台再用更多线程从缓冲队列消费，处理结果自行 ack 或放入待确认的缓冲队列，再由独立线程进行消息确认。

一、Rabbit 自动配置

@Configuration
@ConditionalOnProperty("spring.rabbitmq.url")
@EnableConfigurationProperties(AmqpProperties.class)
@Import(RabbitAutoConfiguration.class)
public class AmqpConfiguration {

    @Autowired
    private AmqpProperties amqpProperties;

    /**
     * Amqp 消费群组
     * <p>
     *
     * @param connectionFactory
     * @return
     * @throws IOException
     */
    @Bean
    public AmqpCoreListenerGroup amqpCoreListenerGroup(ConnectionFactory connectionFactory) throws IOException {
        final Connection conn = connectionFactory.createConnection();
        AmqpCoreListenerGroup amqpCoreListenerGroup = new AmqpCoreListenerGroup();
        amqpProperties.getQueues().forEach((queue, consumer) -> {
            final Channel channel = conn.createChannel(Boolean.FALSE);
            amqpCoreListenerGroup.add(new AmqpCoreListener(channel, queue, consumer, amqpProperties).init());
        });
        return amqpCoreListenerGroup;
    }
}

二、核心 Listener

1、ListenerGroup

package com.xugy.apprabbit.module.amqp.core;

import com.xugy.apprabbit.module.amqp.core.AmqpCoreListener;

import java.util.ArrayList;
import java.util.List;

/**
 * Created by xuguangyuansh on 2017/10/11.
 */
public class AmqpCoreListenerGroup {
    private List<AmqpCoreListener> amqpCoreListenerGroup = new ArrayList<AmqpCoreListener>();

    public void add(AmqpCoreListener amqpCoreListener) {
        this.amqpCoreListenerGroup.add(amqpCoreListener);
    }

    public List<AmqpCoreListener> getAmqpCoreListenerGroup() {
        return amqpCoreListenerGroup;
    }

    public void setAmqpCoreListenerGroup(List<AmqpCoreListener> amqpCoreListenerGroup) {
        this.amqpCoreListenerGroup = amqpCoreListenerGroup;
    }
}

2、CoreListener

package com.xugy.apprabbit.module.amqp.core;

import com.rabbitmq.client.*;
import com.xugy.apprabbit.boot.autoconfigure.amqp.AmqpProperties;
import com.xugy.apprabbit.module.amqp.AmqpEntry;
import com.xugy.apprabbit.module.amqp.Messagelistener;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.amqp.core.Message;

import java.io.IOException;
import java.util.Arrays;
import java.util.concurrent.*;

/**
 * @Author xuguangyuansh@126.com
 */
public class AmqpCoreListener {
    protected static final Logger logger = LoggerFactory.getLogger(AmqpCoreListener.class);

    protected static final ExecutorService executor = Executors.newCachedThreadPool();

    protected final ArrayBlockingQueue<AmqpEntry> actionQueue;
    protected final ArrayBlockingQueue<AmqpEntry> callbackQueue;

    protected final AmqpProperties amqpProperties;
    protected final Channel channel;
    protected final String consumer;
    protected final String queue;

    public AmqpCoreListener(Channel channel, String queue, String consumer, AmqpProperties amqpProperties) {
        this.actionQueue = new ArrayBlockingQueue(amqpProperties.getQos());
        this.callbackQueue = new ArrayBlockingQueue(amqpProperties.getQos());
        this.amqpProperties = amqpProperties;
        this.channel = channel;
        this.queue = queue;
        this.consumer = consumer;
    }

    public AmqpCoreListener init() {
        listen();
        consume();
        ackOrNack();
        return this;
    }

    /***********************************************/
    /********            建立监听            ********/
    /***********************************************/
    protected void listen() {
        executor.execute(() -> {
            try {
                channel.basicQos(amqpProperties.getQos());
            } catch (IOException e) {
                throw new RuntimeException(">>>>> AmqpCoreListener init: when basicQos Exception: {}", e);
            }
            final Consumer consumer = new DefaultConsumer(channel) {
                @Override
                public void handleDelivery(String consumerTag, Envelope envelope, AMQP.BasicProperties properties, byte[] body) throws IOException {
                    if (!Thread.currentThread().isInterrupted()) {
                        Message message = new Message(body, null);
                        AmqpEntry amqpEntry = new AmqpEntry(message, envelope);
                        try {
                            actionQueue.put(amqpEntry);
                        } catch (InterruptedException e) {
                            Thread.currentThread().interrupt();
                            logger.warn(">>>>> [amqp listener thread] An interrupt signal is received, but AMQP callbackqueue is not empty and will be delayed to exit.");
                        }
                    }
                }
            };
            try {
                channel.basicConsume(queue, false, consumer);
            } catch (IOException e) {
                logger.warn(">>>>> AmqpCoreListener basicConsume Exception: ");
                e.printStackTrace();
            }
        });
    }

    /***********************************************/
    /********            消息消费            ********/
    /***********************************************/
    protected void consume() {
        Messagelistener messageListener;
        try {
            messageListener = (Messagelistener) Class.forName(consumer).getDeclaredConstructor().newInstance();
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
        Arrays.stream(new int[amqpProperties.getCurrent()]).forEach(i -> {
            executor.execute(() -> {
                messageListener.setCallbackQueue(callbackQueue);
                while (!Thread.currentThread().isInterrupted()) {
                    AmqpEntry amqpEntry = null;
                    try {
                        amqpEntry = actionQueue.take();
                    } catch (InterruptedException e) {
                        Thread.currentThread().interrupt();
                        logger.warn(">>>>> [amqp consume thread] An interrupt signal is received, will exit.");
                        continue;
                    }
                    try {
                        messageListener.onMessage(amqpEntry);
                        amqpEntry.setAck(Boolean.TRUE);
                    } catch (Exception e) {
                        amqpEntry.setAck(Boolean.FALSE);
                        e.printStackTrace();
                    } finally {
                        if (amqpProperties.isAutoAck())
                            while (!callbackQueue.offer(amqpEntry)) ;
                    }
                }
            });

        });
    }

    /***********************************************/
    /********            消息确认            ********/
    /***********************************************/
    protected void ackOrNack() {
        executor.execute(() -> {
            while (true) {
                try {
                    AmqpEntry amqpEntry = callbackQueue.poll(300, TimeUnit.MILLISECONDS);
                    if (amqpEntry == null) {
                        if (Thread.currentThread().isInterrupted()) {
                            if (((ThreadPoolExecutor) executor).getActiveCount() < 3)
                                break;
                            else
                                logger.warn(">>>>> [amqp ack thread] An interrupt signal is received, but [amqp consume thread] has not over and will be delayed to exit.");
                        }
                    } else if (amqpEntry.isAck())
                        channel.basicAck(amqpEntry.getEnvelope().getDeliveryTag(), false);
                    else
                        channel.basicNack(amqpEntry.getEnvelope().getDeliveryTag(), false, true);
                } catch (InterruptedException e) {
                    Thread.currentThread().interrupt();
                    logger.warn(">>>>> [amqp ack thread] An interrupt signal is received, but AMQP callbackqueue is not empty and will be delayed to exit.");
                } catch (NullPointerException e) {
                    e.printStackTrace();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        });
    }
}

三、抽象 Listener

package com.xugy.apprabbit.module.amqp;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.concurrent.ArrayBlockingQueue;

/**
 * Created by xuguangyuansh on 2017/10/11.
 */
public abstract class AbstractMessageListener implements Messagelistener {
    private final Logger logger = LoggerFactory.getLogger(getClass());

    public ArrayBlockingQueue<AmqpEntry> callbackQueue;

    public void Ack(AmqpEntry amqpEntry) {
        try {
            callbackQueue.put(amqpEntry.setAck(Boolean.TRUE));
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            logger.warn(">>>>> An interrupt signal is received, but AMQP callbackqueue is not empty and will be delayed to exit.");
        }
    }

    public void Nack(AmqpEntry amqpEntry) {
        try {
            callbackQueue.put(amqpEntry.setAck(Boolean.FALSE));
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            logger.warn(">>>>> An interrupt signal is received, but AMQP callbackqueue is not empty and will be delayed to exit.");
        }
    }

    @Override
    public abstract void onMessage(AmqpEntry amqpEntry);

    @Override
    public void setCallbackQueue(ArrayBlockingQueue<AmqpEntry> callbackQueue) {
        this.callbackQueue = callbackQueue;
    }
}

四、业务 Listener

package com.xugy.apprabbit.module;

import com.xugy.apprabbit.module.amqp.AbstractMessageListener;
import com.xugy.apprabbit.module.amqp.AmqpEntry;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.amqp.utils.SerializationUtils;

/**
 * Created by xuguangyuansh on 2017/10/11.
 */
public class SyncMessageListener extends AbstractMessageListener {
    private final Logger logger = LoggerFactory.getLogger(SyncMessageListener.class);

    @Override
    public void onMessage(AmqpEntry amqpEntry) {
        String routingkey = amqpEntry.getEnvelope().getRoutingKey();
        Object object = SerializationUtils.deserialize(amqpEntry.getMessage().getBody());
    }
}