动态订阅mq实现(消费者组动态上下线)

最新推荐文章于 2024-03-13 09:43:56 发布

tomatooooooooooo

最新推荐文章于 2024-03-13 09:43:56 发布

阅读量481

点赞数

文章标签： java rocketmq

本文链接：https://blog.csdn.net/billiefan/article/details/132166774

版权

文章描述了一种在Java中使用Spring和SLF4J实现的MySQLbinlog消费器，它支持动态添加和删除消费者组，每个topic对应一个消费者组，且消费者组需幂等创建。代码展示了如何初始化消费者映射、添加和删除消费者以及处理消息消费逻辑。

摘要由CSDN通过智能技术生成

需求

消费mysql binlog Mq，现状是固定前缀的多topic mq，实现已配置的topic订阅消费，未配置的不消费

实现

一个topic一个消费者组，项目启动时加载订阅并启动，已配置的topic，项目启动后，可以人为操作增加订阅或取消订阅
注意：一个消费者组只能启动一次，所以在增加消费者组时需要幂等

代码


@Component
@Slf4j
public class BinlogConsumerInit {

    /**
     * Consumer集合
     */
    private static final Map<String, DefaultMQPushConsumer> consumerMap = new ConcurrentHashMap<>();

    /**
     * topic固定前缀
     */
    private static final String IDB_BINLOG_MQ_PREFIX = "CDCMQ";

    /**
     * 消费者组名称统一前缀
     */
    private static final String CONSUMER_GROUP_PREFIX = "datasync_binlog_sub_group_";


    @Resource
    BinlogPortDatabaseConfigMapper binlogPortDatabaseConfigMapper;

    @Resource
    Consumer<BinlogEvent> tConsumer;


    /**
     * 初始化加载所有consumer
     */
    @PostConstruct
    public void initConsumerMap() {
        log.info("initConsumerMap, 初始化consumerMap start");
        List<BinlogPortDatabaseConfig> binlogPortDatabaseConfigs = binlogPortDatabaseConfigMapper.selectConfigByStatus(BinlogDataSyncEnums.configStatus.USED.getConfigStatus());
        if (binlogPortDatabaseConfigs.size() == 0) {
            return;
        }
        try {
            for (BinlogPortDatabaseConfig config : binlogPortDatabaseConfigs) {
                consumerAdd(IDB_BINLOG_MQ_PREFIX + config.getProPort());
            }
        } catch (Exception e) {
            log.error("initConsumerMap, 初始化consumerMap异常");
            e.printStackTrace();
        }
    }

    /**
     * 消费者组上线
     *
     * @param topic
     * @throws Exception
     */
    public void consumerAdd(String topic) throws MQClientException {
        if (consumerMap.containsKey(topic)) {
            return;
        }
        String port = geProPortByTopic(topic);
        // 一个端口号一个消费者组
        DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(CONSUMER_GROUP_PREFIX + port);
        // 订阅topic所有消息
        consumer.subscribe(topic, SubscriptionData.SUB_ALL);
        consumer.registerMessageListener((MessageListenerConcurrently) (msgs, context) -> {
            for (MessageExt msg : msgs) {
                log.info("消费binlog mq, topic={}, msgId={}, msg={}", topic, msg.getMsgId(), msg);
                String msgBody = new String(msg.getBody(), StandardCharsets.UTF_8);
                BinlogEvent binlogEvent = JsonUtil.silentString2Object(msgBody, BinlogEvent.class);
                binlogEvent.setPort(Long.valueOf(port));
                //  具体消费逻辑
                tConsumer.consume(binlogEvent);
            }
            return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
        });
        // 启动消费者实例
        consumer.start();
        consumerMap.put(topic, consumer);
    }

    /**
     * 消费者组下线
     *
     * @param topic
     */
    public void consumerDelete(String topic) {
        DefaultMQPushConsumer consumer = consumerMap.get(topic);
        if (consumer != null) {
            consumer.shutdown();
            consumerMap.remove(topic);
        }
    }

    /**
     * 通过topic切割端口号
     */
    private String geProPortByTopic(String topic) {
        if (StringUtils.isEmpty(topic)) {
            return null;
        }
        return topic.replace(IDB_BINLOG_MQ_PREFIX, "");
    }

}