Rabbit MQ使用实践

Rabbit MQ使用

1、配置

1.1 pom配置

引入rabbit mq的依赖

      <!--rabbitmq-->
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-amqp</artifactId>
            <exclusions>
                <exclusion>
                    <groupId>org.springframework.boot</groupId>
                    <artifactId>spring-boot-starter-logging</artifactId>
                </exclusion>
            </exclusions>
        </dependency>

1.2 application.yml文件配置

spring:
  # MQ配置
  rabbitmq:
    host: localhost
    port: 5672
    username: xlt
    password: xlt
    # 虚拟host 可以不设置,使用server默认host
    virtual-host: XltHost
    listener:
      simple:
        acknowledge-mode: manual #设置确认模式手工确认
        concurrency: 3 #消费者最小数量
        max-concurrency: 10 # 消费者最大数量
        retry:
          #开启消费者（程序出现异常的情况下会）进行重试
          enabled: true
          #最大重试次数
          max-attempts: 3
          #重试间隔次数
          initial-interval: 300
          #开启手动ack
          #acknowledge-mode: manual

1.3 直连模式配置

//direct直连模式的交换机配置,包括一个direct交换机，两个队列，三根网线binding
@Configuration
public class DirectExchangeConfig {

    @Bean
    public DirectExchange directExchange() {
        DirectExchange directExchange = new DirectExchange("directExchange");
        return directExchange;
    }

    @Bean
    public Queue directQueue1() {
        Queue queue = new Queue("directQueue1");
        return queue;
    }

    @Bean
    public Queue directQueue2() {
        Queue queue = new Queue("directQueue2");
        return queue;
    }

    //3个binding将交换机和相应队列连起来
    @Bean
    public Binding bindingorange() {
        Binding binding = BindingBuilder.bind(directQueue1()).to(directExchange()).with("orange");
        return binding;
    }

    @Bean
    public Binding bindingblack() {
        Binding binding = BindingBuilder.bind(directQueue2()).to(directExchange()).with("black");
        return binding;
    }

    @Bean
    public Binding bindinggreen() {
        Binding binding = BindingBuilder.bind(directQueue2()).to(directExchange()).with("green");
        return binding;
    }
}

1.4 topic队列配置

package com.xxx.xlt.config.mq;

import org.springframework.amqp.core.Binding;
import org.springframework.amqp.core.BindingBuilder;
import org.springframework.amqp.core.Queue;
import org.springframework.amqp.core.TopicExchange;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

//topic交换机模型，需要一个topic交换机，两个队列和三个binding
@Configuration
public class TopicExchangeConfig {
    @Bean
    public TopicExchange topicExchange(){
        TopicExchange topicExchange=new TopicExchange("myTopicExchange");
        return topicExchange;
    }

    @Bean
    public Queue topicQueue1() {
        Queue queue=new Queue("topicQueue1");
        return queue;
    }

    @Bean
    public Queue topicQueue2() {
        Queue queue=new Queue("topicQueue2");
        return queue;
    }

    //3个binding将交换机和相应队列连起来
    @Bean
    public Binding bindingtopic1(){
        Binding binding= BindingBuilder.bind(topicQueue1()).to(topicExchange()).with("*.orange.*");//binding key
        return binding;
    }

    @Bean
    public Binding bindingtopic2(){
        Binding binding= BindingBuilder.bind(topicQueue2()).to(topicExchange()).with("*.*.rabbit");
        return binding;
    }

    @Bean
    public Binding bindingtopic3(){
        Binding binding= BindingBuilder.bind(topicQueue2()).to(topicExchange()).with("lazy.#");//#表示0个或若干个关键字，*表示一个关键字
        return binding;
    }
}

1.5 广播队列

package com.xxx.xlt.config.mq;

import org.springframework.amqp.core.Binding;
import org.springframework.amqp.core.BindingBuilder;
import org.springframework.amqp.core.FanoutExchange;
import org.springframework.amqp.core.Queue;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

/**
 * 发布订阅模式的配置,包括两个队列和对应的订阅者,发布者的交换机类型使用fanout(子网广播),两根网线binding用来绑定队列到交换机
 */
@Configuration
public class PublishSubscribeConfig {

    @Bean
    public Queue myQueue1() {
        Queue queue = new Queue("queue1");
        return queue;
    }

    @Bean
    public Queue myQueue2() {
        Queue queue = new Queue("queue2");
        return queue;
    }

    @Bean
    public FanoutExchange fanoutExchange() {
        FanoutExchange fanoutExchange = new FanoutExchange("fanout");
        return fanoutExchange;
    }

    @Bean
    public Binding binding1() {
        Binding binding = BindingBuilder.bind(myQueue1()).to(fanoutExchange());
        return binding;
    }

    @Bean
    public Binding binding2() {
        Binding binding = BindingBuilder.bind(myQueue2()).to(fanoutExchange());
        return binding;
    }

}

1.6 简单队列

@Configuration
public class ProducerConsumerConfig {
    @Bean
    public Queue myQueue() {
        Queue queue = new Queue("myqueue");
        return queue;
    }
}

2、消息发送

发送消息线程类MqSendThread

package com.xxx.xlt.service.mq.producer;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.amqp.rabbit.core.RabbitTemplate;

public class MqSendThread implements Runnable{
    private static final Logger logger= LoggerFactory.getLogger(MqSender.class);

    private String exchange;
    private String routeKey;
    private Object msg;
    private RabbitTemplate rabbitTemplate;

    public MqSendThread(String exchange,String routeKey,Object msg,RabbitTemplate rabbitTemplate) {
        this.exchange=exchange;
        this.routeKey=routeKey;
        this.msg=msg;
        this.rabbitTemplate= rabbitTemplate;
    }

    @Override
    public void run() {
        logger.info("Current threadId={} threadName={}",Thread.currentThread().getId(),Thread.currentThread().getName());
        sendTopicMsg(exchange,routeKey,msg);

    }

    public void sendTopicMsg(String exchange,String routeKey,Object msg) {
        rabbitTemplate.convertAndSend(exchange,routeKey,msg);
        logger.info("threadName={} sends topic exchange Mq msg with multiple thread successfully",Thread.currentThread().getName());
    }
}

发消息

package com.xxx.xlt.service.mq.producer;


import com.alibaba.fastjson.JSON;
import com.xxx.xlt.utils.common.SnowflakeIdGenerator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.amqp.core.Message;
import org.springframework.amqp.core.MessageProperties;
import org.springframework.amqp.core.ReturnedMessage;
import org.springframework.amqp.rabbit.connection.CorrelationData;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;

@Service
public class MqSender implements RabbitTemplate.ConfirmCallback,RabbitTemplate.ReturnsCallback{
    private static final Logger logger= LoggerFactory.getLogger(MqSender.class);

    @Autowired
    private RabbitTemplate rabbitTemplate;

    public void sendDirect(Object msg) {
        logger.info("send direct Mq msg");
        rabbitTemplate.convertAndSend("directExchange","orange",msg);
        logger.info("send direct Mq msg successfully");
    }

    public void sendTopic(Object msg) {
        logger.info("send topic exchange Mq msg");
        Message message = preHandleMsg(msg);

        for (int i=0;i<4;i++) {
            rabbitTemplate.convertAndSend("myTopicExchange","*.orange.*",message);
        }
        logger.info("send topic exchange Mq msg successfully");
    }

    public void sendTopicMulti(Object msg) {
        //创建等待队列
        BlockingQueue<Runnable> bqueue = new ArrayBlockingQueue<Runnable>(20);
        //创建线程池，池中保存的线程数为3，允许的最大线程数为5
        ThreadPoolExecutor pool = new ThreadPoolExecutor(5,10,50, TimeUnit.MILLISECONDS,bqueue);
        logger.info("send topic exchange Mq msg with multiple thread");
        Message message = preHandleMsg(msg);
        Runnable t1=new MqSendThread("myTopicExchange","*.orange.*",message,rabbitTemplate);
        Runnable t2=new MqSendThread("myTopicExchange","*.orange.*",message,rabbitTemplate);
        Runnable t3=new MqSendThread("myTopicExchange","*.orange.*",message,rabbitTemplate);
        Runnable t4=new MqSendThread("myTopicExchange","*.orange.*",message,rabbitTemplate);
        Runnable t5=new MqSendThread("myTopicExchange","*.orange.*",message,rabbitTemplate);
        pool.execute(t1);
        pool.execute(t2);
        pool.execute(t3);
        pool.execute(t4);
        pool.execute(t5);
        logger.info("send topic exchange Mq msg successfully");
    }

    private Message preHandleMsg(Object msg) {
        MessageProperties properties = new MessageProperties();
        properties.setMessageId(SnowflakeIdGenerator.generateId().toString());
        byte[] msgBytes = JSON.toJSONBytes(msg);
        Message message = new Message(msgBytes, properties);
        rabbitTemplate.setConfirmCallback(this);
        rabbitTemplate.setReturnsCallback(this);
        return message;
    }

    @Override
    public void confirm(CorrelationData correlationData, boolean ack, String cause) {
        logger.info("Confirm: " + correlationData + ", ack=" + ack + (cause == null ? "" : ", cause: " + cause));
    }

    @Override
    public void returnedMessage(ReturnedMessage returnedMessage) {
          logger.info("returned msg is={}",returnedMessage);
    }
}

3、消费消息

防止消息重复消费

3.1单体部署

考虑多线程的情况下

@Component
public class MqConsumer {
    private static final Logger logger = LoggerFactory.getLogger(MqConsumer.class);
    private static final ReentrantLock lock = new ReentrantLock(true); // 公平锁

    @RabbitListener(queues = "topicQueue1")
    @RabbitHandler
    void topicHandler1(Message msg, @Headers Map<String, Object> headers, Channel channel) throws IOException, InterruptedException {
        String messageId = msg.getMessageProperties().getMessageId();
        if (StringUtils.isEmpty(messageId)) {
            throw new CommonException("MessageID  is empty");
        }
        logger.info("receive message id is {}", msg.getMessageProperties().getMessageId());
        if (lock.tryLock(5, TimeUnit.SECONDS)) { // 获取锁的时间与业务处理时间成正比，一般要大于业务处理的时间
            try {
                Boolean absent = RedisUtil.setIfAbsent(messageId, 1, Duration.ofSeconds(60));
                if (!absent) {
                    logger.info("Repeat for [{}]", messageId);
                    return;  //直接返回即可，不要抛出异常，因为MQ有自动重试机制，抛出异常之后，MQ任务消息没有处理成功，会多次进行重试
                }
                logger.info("msgID={}, body={}", msg.getMessageProperties().getMessageId(), JSON.parse(msg.getBody()));
            } catch (Exception e) {
                RedisUtil.del(messageId);  // 业务处理失败需要把Redis里面的消息删掉，否则后面重试会在业务处理之前被拦截，达不到重试的目的
                logger.error("Error appears when process business:", e);
                throw e; // 必须把异常抛出来，否则业务处理失败，MQ不会进行重试
            } finally {
                ackSuccess(headers, channel); // 处理成功，应答一下
                lock.unlock();
                logger.info("unlock success");
            }
        }
    }

    private void ackSuccess(@Headers Map<String, Object> headers, Channel channel) throws IOException {
        long deliveryTag = (long) headers.get(AmqpHeaders.DELIVERY_TAG);
        channel.basicAck(deliveryTag, true);
    }

    @RabbitListener(queues = "topicQueue2")
    @RabbitHandler
    void topicHandler2(Object msg) {
        logger.info("receive message is {}", msg.toString());
    }

    @RabbitListener(queues = "directQueue1")
    @RabbitHandler
    void directHandler1(Object msg) {
        logger.info("receive message is {}", msg.toString());
    }

    @RabbitListener(queues = "directQueue2")
    @RabbitHandler
    void directHandler2(Object msg) {
        logger.info("receive message is {}", msg.toString());
    }
}

3.2集群部署

这时可能多个实例同时消费消息，需要使用分布式锁

@Component
public class MqConsumer {
    private static final Logger logger = LoggerFactory.getLogger(MqConsumer.class);
//    private static final ReentrantLock lock = new ReentrantLock(true); // 公平锁

    @Autowired
    private RedissonClient redissonClient; // 分布式锁

    @Autowired
    private HttpUtil httpUtil;

    @RabbitListener(queues = "topicQueue1")
    @RabbitHandler
    void topicHandler1(Message msg, @Headers Map<String, Object> headers, Channel channel) throws IOException, InterruptedException {
        String messageId = msg.getMessageProperties().getMessageId();
        if (StringUtils.isEmpty(messageId)) {
            throw new CommonException("MessageID  is empty");
        }
        logger.info("receive message id is {}", msg.getMessageProperties().getMessageId());
        Lock lock = redissonClient.getLock(messageId + "_lock");
        if (lock.tryLock(5, TimeUnit.SECONDS)) { // 获取分布式锁的时间与业务处理时间成正比，一般要大于业务处理的时间
            try {
                Boolean absent = RedisUtil.setIfAbsent(messageId, 1, Duration.ofSeconds(60));
                if (!absent) {
                    logger.info("Repeat for [{}]", messageId);
                    return;  //直接返回即可，不要抛出异常，因为MQ有自动重试机制，抛出异常之后，MQ任务消息没有处理成功，会多次进行重试
                }
                logger.info("msgID={}, body={}", msg.getMessageProperties().getMessageId(), JSON.parse(msg.getBody()));
            } catch (Exception e) {
                RedisUtil.del(messageId);  // 业务处理失败需要把Redis里面的消息删掉，否则后面重试会在业务处理之前被拦截，达不到重试的目的
                logger.error("Error appears when process business:", e);
                throw e; // 必须把异常抛出来，否则业务处理失败，MQ不会进行重试
            } finally {
                ackSuccess(headers, channel); // 处理成功，应答一下
                lock.unlock();
                logger.info("unlock success");
            }
        }
    }

    private void ackSuccess(@Headers Map<String, Object> headers, Channel channel) throws IOException {
        long deliveryTag = (long) headers.get(AmqpHeaders.DELIVERY_TAG);
        channel.basicAck(deliveryTag, true);
    }

    @RabbitListener(queues = "topicQueue2")
    @RabbitHandler
    void topicHandler2(Object msg) {
        logger.info("receive message is {}", msg.toString());
    }

    @RabbitListener(queues = "directQueue1")
    @RabbitHandler
    void directHandler1(Object msg) {
        logger.info("receive message is {}", msg.toString());
    }

    @RabbitListener(queues = "directQueue2")
    @RabbitHandler
    void directHandler2(Object msg) {
        logger.info("receive message is {}", msg.toString());
    }
}