第十六章-消费者-PUSH方式（一）

最新推荐文章于 2024-10-28 20:09:57 发布

多栖码农

最新推荐文章于 2024-10-28 20:09:57 发布

阅读量198

点赞数 1

分类专栏： RocketMQ源码文章标签： java rocketmq

本文链接：https://blog.csdn.net/zhang527294844/article/details/138813044

版权

RocketMQ源码专栏收录该内容

23 篇文章 1 订阅

订阅专栏

16.1 准备阶段

先从一段官方示例代码开始

public class Consumer {
    public static void main(String[] args) throws InterruptedException, MQClientException {
        // 初始化consumer，并设置consumer group name
        DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("please_rename_unique_group_name");

        // 设置NameServer地址 
        consumer.setNamesrvAddr("localhost:9876");
        //订阅一个或多个topic，并指定tag过滤条件，这里指定*表示接收所有tag的消息
        consumer.subscribe("TopicTest", "*");
        //注册回调接口来处理从Broker中收到的消息
        consumer.registerMessageListener(new MessageListenerConcurrently() {
            @Override
            public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs, ConsumeConcurrentlyContext context) {
                System.out.printf("%s Receive New Messages: %s %n", Thread.currentThread().getName(), msgs);
                // 返回消息消费状态，ConsumeConcurrentlyStatus.CONSUME_SUCCESS为消费成功
                return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
            }
        });
        // 启动Consumer
        consumer.start();
        System.out.printf("Consumer Started.%n");
    }
}

更多的介绍，可以直接看官网，咱们这直接上源码，那就从 consumer.start() 这一行代码开始。

DefaultMQPushConsumer.start

public void start() throws MQClientException {
    // 通过命名空间处理一下消费者组
    setConsumerGroup(NamespaceUtil.wrapNamespace(this.getNamespace(), this.consumerGroup));
    // push消费的实现类启动
    this.defaultMQPushConsumerImpl.start();
    // 访问日志追踪
    if (null != traceDispatcher) {
        try {
            traceDispatcher.start(this.getNamesrvAddr(), this.getAccessChannel());
        } catch (MQClientException e) {
            log.warn("trace dispatcher start failed ", e);
        }
    }
}

DefaultMQPushConsumerImpl.start

public synchronized void start() throws MQClientException {
    switch (this.serviceState) {
        case CREATE_JUST: // 启动时，看这块
            log.info("the consumer [{}] start beginning. messageModel={}, isUnitMode={}", this.defaultMQPushConsumer.getConsumerGroup(),
                     this.defaultMQPushConsumer.getMessageModel(), this.defaultMQPushConsumer.isUnitMode());
            // 先设置成失败的状态
            this.serviceState = ServiceState.START_FAILED;
			// 检查配置项
            this.checkConfig();
		   // 复制订阅信息，这里所做的事就是将订阅信息从 DefaultMQPushConsumerImpl 类的map 复制到 RebalanceImpl 类的map（rebalanceImpl.getSubscriptionInner()）中，因为最终做消息处理的是 RebalanceImpl去与服务端打交道
            this.copySubscription();
			// 集群模式，实例名改成对应的进程Id
            if (this.defaultMQPushConsumer.getMessageModel() == MessageModel.CLUSTERING) {
                this.defaultMQPushConsumer.changeInstanceNameToPID();
            }
			// 获取或创建MQClientInstance实例，这个类很重要，同时涵盖了生产者和消费者的操作，具体使用时会讲
            this.mQClientFactory = MQClientManager.getInstance().getAndCreateMQClientInstance(this.defaultMQPushConsumer, this.rpcHook);
			//设置消费者组名
            this.rebalanceImpl.setConsumerGroup(this.defaultMQPushConsumer.getConsumerGroup());
            //设置消费模式
            this.rebalanceImpl.setMessageModel(this.defaultMQPushConsumer.getMessageModel());
             //设置分配mq策略（决定消费哪个mq的内容）       
            this.rebalanceImpl.setAllocateMessageQueueStrategy(this.defaultMQPushConsumer.getAllocateMessageQueueStrategy());
            //设置mq客户端实例对象
            this.rebalanceImpl.setmQClientFactory(this.mQClientFactory);
			//创建实际拉取消息的对象PullAPIWrapper
            this.pullAPIWrapper = new PullAPIWrapper(
                mQClientFactory,
                this.defaultMQPushConsumer.getConsumerGroup(), isUnitMode());
            //注册消息过滤处理器列表
            this.pullAPIWrapper.registerFilterMessageHook(filterMessageHookList);
			//先在本地缓存中获取offsetstore，如果没有就去远程broker获取，offsetstore其实就是存放消费者消费mq的偏移量，防止重复消费
            if (this.defaultMQPushConsumer.getOffsetStore() != null) {
                this.offsetStore = this.defaultMQPushConsumer.getOffsetStore();
            } else {
                switch (this.defaultMQPushConsumer.getMessageModel()) {
                    case BROADCASTING:// 广播模式，不做讲解
                        this.offsetStore = new LocalFileOffsetStore(this.mQClientFactory, this.defaultMQPushConsumer.getConsumerGroup());
                        break;
                    case CLUSTERING://集群模式
                        // 集群模式下，用RemoteBrokerOffsetStore对象来处理消费偏移
                        this.offsetStore = new RemoteBrokerOffsetStore(this.mQClientFactory, this.defaultMQPushConsumer.getConsumerGroup());
                        break;
                    default:
                        break;
                }
                this.defaultMQPushConsumer.setOffsetStore(this.offsetStore);
            }
            // 加载本机偏移信息，集群模式用的是 RemoteBrokerOffsetStore 类对象，默认load没有实现内容，这是一个钩子函数，子类可自行实现
            this.offsetStore.load();
			// 决定消息是顺序的还是并发的消费
            if (this.getMessageListenerInner() instanceof MessageListenerOrderly) {// 顺序
                this.consumeOrderly = true;
                this.consumeMessageService =
                    new ConsumeMessageOrderlyService(this, (MessageListenerOrderly) this.getMessageListenerInner());
            } else if (this.getMessageListenerInner() instanceof MessageListenerConcurrently) {// 并发
                this.consumeOrderly = false;
                this.consumeMessageService =
                    new ConsumeMessageConcurrentlyService(this, (MessageListenerConcurrently) this.getMessageListenerInner());
            }
			// 这里主要是对过期消息的任务处理
            this.consumeMessageService.start();
//将消费实例对象注册到本地内存consumerTable（ConcurrentMap）中，一个消费者组在一个客户端内，只会有一个消费实例对象
            boolean registerOK = mQClientFactory.registerConsumer(this.defaultMQPushConsumer.getConsumerGroup(), this);
            if (!registerOK) {
                //注册失败，恢复状态为创建，并向上抛出异常
                this.serviceState = ServiceState.CREATE_JUST;
                this.consumeMessageService.shutdown();
                throw new MQClientException("The consumer group[" + this.defaultMQPushConsumer.getConsumerGroup()
                                            + "] has been created before, specify another name please." + FAQUrl.suggestTodo(FAQUrl.GROUP_NAME_DUPLICATE_URL),
                                            null);
            }
			//上面都是一些检查和数据准备工作，这一步才是真正启动服务的工作，启动内部实现，后面也会讲
            mQClientFactory.start();
            log.info("the consumer [{}] start OK.", this.defaultMQPushConsumer.getConsumerGroup());
            this.serviceState = ServiceState.RUNNING;
            break;
        case RUNNING:
        case START_FAILED:
        case SHUTDOWN_ALREADY:
            throw new MQClientException("The PushConsumer service state not OK, maybe started once, "
                                        + this.serviceState
                                        + FAQUrl.suggestTodo(FAQUrl.CLIENT_SERVICE_NOT_OK),
                                        null);
        default:
            break;
    }
	// 从namesrv中更新topic的路由信息
    this.updateTopicSubscribeInfoWhenSubscriptionChanged();
    // 通过Broker验证客户端的订阅数据合法性
    this.mQClientFactory.checkClientInBroker();
    // 发送心跳
    this.mQClientFactory.sendHeartbeatToAllBrokerWithLock();
    // 启动完成后，唤醒 RebalanceService 服务
    this.mQClientFactory.rebalanceImmediately();
}

checkConfig 检查配置项

private void checkConfig() throws MQClientException {
    // group名 合法性检查
    Validators.checkGroup(this.defaultMQPushConsumer.getConsumerGroup());
	// group判空，个人感觉这里的判断完全可以放在前面
    if (null == this.defaultMQPushConsumer.getConsumerGroup()) {
        throw new MQClientException(
            "consumerGroup is null"
            + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL),
            null);
    }
	// group 不能是默认的名字
    if (this.defaultMQPushConsumer.getConsumerGroup().equals(MixAll.DEFAULT_CONSUMER_GROUP)) {
        throw new MQClientException(
            "consumerGroup can not equal "
            + MixAll.DEFAULT_CONSUMER_GROUP
            + ", please specify another one."
            + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL),
            null);
    }
	// 消息模式不能为空，只有2种：集群和广播
    if (null == this.defaultMQPushConsumer.getMessageModel()) {
        throw new MQClientException(
            "messageModel is null"
            + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL),
            null);
    }
	// 判断从哪开始消费，默认从最后一次偏移，包括最后一次偏移、最小偏移、最大偏移等
    if (null == this.defaultMQPushConsumer.getConsumeFromWhere()) {
        throw new MQClientException(
            "consumeFromWhere is null"
            + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL),
            null);
    }
	// 时间格式化
    Date dt = UtilAll.parseDate(this.defaultMQPushConsumer.getConsumeTimestamp(), UtilAll.YYYYMMDDHHMMSS);
    if (null == dt) {
        throw new MQClientException(
            "consumeTimestamp is invalid, the valid format is yyyyMMddHHmmss,but received "
            + this.defaultMQPushConsumer.getConsumeTimestamp()
            + " " + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), null);
    }

    // 消息队列分配策略，默认是平均分配
    if (null == this.defaultMQPushConsumer.getAllocateMessageQueueStrategy()) {
        throw new MQClientException(
            "allocateMessageQueueStrategy is null"
            + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL),
            null);
    }

    // 订阅容器map，不为空
    if (null == this.defaultMQPushConsumer.getSubscription()) {
        throw new MQClientException(
            "subscription is null"
            + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL),
            null);
    }

    // push方式是异步的，所以需要一个消息监听器，因此，这里不能为空
    if (null == this.defaultMQPushConsumer.getMessageListener()) {
        throw new MQClientException(
            "messageListener is null"
            + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL),
            null);
    }
	// 顺序还是并发类型
    boolean orderly = this.defaultMQPushConsumer.getMessageListener() instanceof MessageListenerOrderly;
    boolean concurrently = this.defaultMQPushConsumer.getMessageListener() instanceof MessageListenerConcurrently;
    // 只能是二者选一，不能两者都不是
    if (!orderly && !concurrently) {
        throw new MQClientException(
            "messageListener must be instanceof MessageListenerOrderly or MessageListenerConcurrently"
            + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL),
            null);
    }

    // 判断最小消费者线程数合法性，默认20
    if (this.defaultMQPushConsumer.getConsumeThreadMin() < 1
        || this.defaultMQPushConsumer.getConsumeThreadMin() > 1000) {
        throw new MQClientException(
            "consumeThreadMin Out of range [1, 1000]"
            + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL),
            null);
    }

    // 判断最大消费者线程数合法性，默认20
    if (this.defaultMQPushConsumer.getConsumeThreadMax() < 1 || this.defaultMQPushConsumer.getConsumeThreadMax() > 1000) {
        throw new MQClientException(
            "consumeThreadMax Out of range [1, 1000]"
            + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL),
            null);
    }

    // consumeThreadMin 不可能比 consumeThreadMax 更大
    if (this.defaultMQPushConsumer.getConsumeThreadMin() > this.defaultMQPushConsumer.getConsumeThreadMax()) {
        throw new MQClientException(
            "consumeThreadMin (" + this.defaultMQPushConsumer.getConsumeThreadMin() + ") "
            + "is larger than consumeThreadMax (" + this.defaultMQPushConsumer.getConsumeThreadMax() + ")",
            null);
    }

    // 判断并发消费最大的偏移跨度，默认2000，非法证明消费过程有问题
    if (this.defaultMQPushConsumer.getConsumeConcurrentlyMaxSpan() < 1
        || this.defaultMQPushConsumer.getConsumeConcurrentlyMaxSpan() > 65535) {
        throw new MQClientException(
            "consumeConcurrentlyMaxSpan Out of range [1, 65535]"
            + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL),
            null);
    }

    // 队列级别的流量控制阈值，默认情况下每个消息队列最多缓存1000条消息，考虑{@code pullBatchSize}，瞬时值可能超过限制
    if (this.defaultMQPushConsumer.getPullThresholdForQueue() < 1 || this.defaultMQPushConsumer.getPullThresholdForQueue() > 65535) {
        throw new MQClientException(
            "pullThresholdForQueue Out of range [1, 65535]"
            + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL),
            null);
    }

    // topic级别的流量控制阈值
    if (this.defaultMQPushConsumer.getPullThresholdForTopic() != -1) {
        if (this.defaultMQPushConsumer.getPullThresholdForTopic() < 1 || this.defaultMQPushConsumer.getPullThresholdForTopic() > 6553500) {
            throw new MQClientException(
                "pullThresholdForTopic Out of range [1, 6553500]"
                + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL),
                null);
        }
    }

    // 队列级别的限制拉取大小，默认情况下每个消息队列最多缓存100M
    if (this.defaultMQPushConsumer.getPullThresholdSizeForQueue() < 1 || this.defaultMQPushConsumer.getPullThresholdSizeForQueue() > 1024) {
        throw new MQClientException(
            "pullThresholdSizeForQueue Out of range [1, 1024]"
            + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL),
            null);
    }
    // topic级别的限制拉取大小，默认情况下每个消息队列最多缓存100M
    if (this.defaultMQPushConsumer.getPullThresholdSizeForTopic() != -1) {
        // pullThresholdSizeForTopic
        if (this.defaultMQPushConsumer.getPullThresholdSizeForTopic() < 1 || this.defaultMQPushConsumer.getPullThresholdSizeForTopic() > 102400) {
            throw new MQClientException(
                "pullThresholdSizeForTopic Out of range [1, 102400]"
                + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL),
                null);
        }
    }

    // 拉取间隔时间
    if (this.defaultMQPushConsumer.getPullInterval() < 0 || this.defaultMQPushConsumer.getPullInterval() > 65535) {
        throw new MQClientException(
            "pullInterval Out of range [0, 65535]"
            + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL),
            null);
    }

    // 批量消费消息的最大大小
    if (this.defaultMQPushConsumer.getConsumeMessageBatchMaxSize() < 1
        || this.defaultMQPushConsumer.getConsumeMessageBatchMaxSize() > 1024) {
        throw new MQClientException(
            "consumeMessageBatchMaxSize Out of range [1, 1024]"
            + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL),
            null);
    }

    // 批量拉取的大小
    if (this.defaultMQPushConsumer.getPullBatchSize() < 1 || this.defaultMQPushConsumer.getPullBatchSize() > 1024) {
        throw new MQClientException(
            "pullBatchSize Out of range [1, 1024]"
            + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL),
            null);
    }
}

MQClientInstance.start

public void start() throws MQClientException {

    synchronized (this) {
        switch (this.serviceState) {
            case CREATE_JUST:
                this.serviceState = ServiceState.START_FAILED;
                //这一步就是获取name server的地址，正常都会指定的，否则没有什么意义
                if (null == this.clientConfig.getNamesrvAddr()) {
                    this.mQClientAPIImpl.fetchNameServerAddr();
                }
 			/* 启动 request-response channel，具体细节不在这里讲，以后有机会再写netty源码，主要做了以下工作
                	1.设置netty工作线程
                	2.设置netty收发消息时的处理器
                	3.创建延迟3秒且每秒执行一次扫描过期无效的reponse信息的请求
                	4.设置消息远程读取/接收的Netty处理器，NettyClientHandler，重点关注，先记着，后面会用
                */
                this.mQClientAPIImpl.start();
                /* 启动rocketmq内置的一些定时任务，如下：
                	1.延迟10秒且每2分钟执行一次获取最新name server 地址的任务
                	2.延迟10毫秒且每{this.clientConfig.getPollNameServerInterval()	     （pollNameServerInterval可配置，但默认是30秒）}毫秒执行一次更新topic路由信息的任务MQClientInstance.this.updateTopicRouteInfoFromNameServer()
                	3.延迟1秒且每{this.clientConfig.getHeartbeatBrokerInterval()	     （heartbeatBrokerInterval可配置，但默认是30秒）}毫秒执行一次清除下线的broker和发送心态到所有broker的任务
                	4.延迟10秒且每{this.clientConfig.getPersistConsumerOffsetInterval()	     （persistConsumerOffsetInterval可配置，但默认是5秒）}毫秒执行一次持久化消息者offset值的任务，持久化存放在哪呢？放到broker中
                	5.延迟1毫秒且每1分钟执行一次动态改变消息推送到消费者的工作线程数的任务
                */
                this.startScheduledTask();
                //启动拉取消息服务
                this.pullMessageService.start();
                // 启动负载均衡服务
                this.rebalanceService.start();
			   //因为MQClientInstance这个类，是生产者和消费都共用的，且消费者也会使用消费生产并发送操作，比如消费失败时重推消息，也需要启动生产者实现类，所以看到这段代码不用奇怪，如果是生产者，就算调用start方法，也不会启动2次，内部有针对状态的判断
                this.defaultMQProducer.getDefaultMQProducerImpl().start(false);
                log.info("the client factory [{}] start OK", this.clientId);
                this.serviceState = ServiceState.RUNNING;
                break;
            case RUNNING:
                break;
            case SHUTDOWN_ALREADY:
                break;
            case START_FAILED:
                throw new MQClientException("The Factory object[" + this.getClientId() + "] has been created before, and failed.", null);
            default:
                break;
        }
    }
}