moquette源码学习

背景

最近一直在看mqtt协议，希望在后续项目中使用一个高并发高可用的mqtt broker，最后发现了moquette。在github上的点赞还是比较多的，所以学习下它的源码以更加熟悉下mqtt的协议。

结构

结构里主要包含broker、interception(拦截器）、logging(日志）和persistence(持久化）
在这里主要分析的是broker

broker

先从server层分析开始。
server类是整个代码的起始类，里面定义了各种接口的实现以及连接工厂的创建。

public void startServer(IConfig config, List<? extends InterceptHandler> handlers, ISslContextCreator sslCtxCreator,
                            IAuthenticator authenticator, IAuthorizatorPolicy authorizatorPolicy) {
        final long start = System.currentTimeMillis();
        if (handlers == null) {
            handlers = Collections.emptyList();
        }
        LOG.trace("Starting Moquette Server. MQTT message interceptors={}", getInterceptorIds(handlers));

        scheduler = Executors.newScheduledThreadPool(1);

        final String handlerProp = System.getProperty(BrokerConstants.INTERCEPT_HANDLER_PROPERTY_NAME);
        if (handlerProp != null) {
            config.setProperty(BrokerConstants.INTERCEPT_HANDLER_PROPERTY_NAME, handlerProp);
        }
        final String persistencePath = config.getProperty(BrokerConstants.PERSISTENT_STORE_PROPERTY_NAME);
        LOG.debug("Configuring Using persistent store file, path: {}", persistencePath);
        initInterceptors(config, handlers);
        LOG.debug("Initialized MQTT protocol processor");
        if (sslCtxCreator == null) {
            LOG.info("Using default SSL context creator");
            sslCtxCreator = new DefaultMoquetteSslContextCreator(config);
        }
        authenticator = initializeAuthenticator(authenticator, config);
        authorizatorPolicy = initializeAuthorizatorPolicy(authorizatorPolicy, config);

        final ISubscriptionsRepository subscriptionsRepository;
        final IQueueRepository queueRepository;
        final IRetainedRepository retainedRepository;
        //判断是否进行持久化处理，如果进行持久化将数据放到h2数据库中
        if (persistencePath != null && !persistencePath.isEmpty()) {
            LOG.trace("Configuring H2 subscriptions store to {}", persistencePath);
            //h2数据库初始化
            h2Builder = new H2Builder(config, scheduler).initStore();
            //建立订阅主题的存储库
            subscriptionsRepository = h2Builder.subscriptionsRepository();
            //创建队列
            queueRepository = h2Builder.queueRepository();
            //持有历史数据库信息
            retainedRepository = h2Builder.retainedRepository();
        } else {
            //如果没定义，那就是用内存保存，断电就丢失了
            LOG.trace("Configuring in-memory subscriptions store");

            //创建内存数据
            subscriptionsRepository = new MemorySubscriptionsRepository();
            queueRepository = new MemoryQueueRepository();
            retainedRepository = new MemoryRetainedRepository();
        }

        /*
        初始化订阅的主题，这里使用了像是二叉树的结构
         */
        ISubscriptionsDirectory subscriptions = new CTrieSubscriptionDirectory();
        subscriptions.init(subscriptionsRepository);

        /*
        判断是否需要认证
         */
        final Authorizator authorizator = new Authorizator(authorizatorPolicy);

        sessions = new SessionRegistry(subscriptions, queueRepository, authorizator);

        dispatcher = new PostOffice(subscriptions, retainedRepository, sessions, interceptor, authorizator);
        final BrokerConfiguration brokerConfig = new BrokerConfiguration(config);
        //创建mqtt连接工厂
        MQTTConnectionFactory connectionFactory = new MQTTConnectionFactory(brokerConfig, authenticator, sessions,
                                                                            dispatcher);
        //netty实现的mqtt 这里处理了netty的连接信息
        //根据连接信息创建
        final NewNettyMQTTHandler mqttHandler = new NewNettyMQTTHandler(connectionFactory);

        //以下开始初始netty启动器
        acceptor = new NewNettyAcceptor();

        acceptor.initialize(mqttHandler, config, sslCtxCreator);

        final long startTime = System.currentTimeMillis() - start;
        LOG.info("Moquette integration has been started successfully in {} ms", startTime);
        initialized = true;
    }

我比较关心订阅的主题和客户端连接信息的存储，所以比较关心持久层的代码
通过下面的代码可以看到，支持两种存储方式，一种是h2数据库，一种是直接存放到内存中
如果项目中有别的需求，直接可以将这里替换成自己想要的数据库，比如redis或者Mysql

 final ISubscriptionsRepository subscriptionsRepository;
        final IQueueRepository queueRepository;
        final IRetainedRepository retainedRepository;
        //判断是否进行持久化处理，如果进行持久化将数据放到h2数据库中
        if (persistencePath != null && !persistencePath.isEmpty()) {
            LOG.trace("Configuring H2 subscriptions store to {}", persistencePath);
            //h2数据库初始化
            h2Builder = new H2Builder(config, scheduler).initStore();
            //建立订阅主题的存储库
            subscriptionsRepository = h2Builder.subscriptionsRepository();
            //创建队列
            queueRepository = h2Builder.queueRepository();
            //持有历史数据库信息
            retainedRepository = h2Builder.retainedRepository();
        } else {
            //如果没定义，那就是用内存保存，断电就丢失了
            LOG.trace("Configuring in-memory subscriptions store");

            //创建内存数据
            subscriptionsRepository = new MemorySubscriptionsRepository();
            queueRepository = new MemoryQueueRepository();
            retainedRepository = new MemoryRetainedRepository();
        }

内存存放

现在再说说存放在内存的方法

public class MemorySubscriptionsRepository implements ISubscriptionsRepository {

    private final List<Subscription> subscriptions = new ArrayList<>();

//不允许修改的List
    @Override
    public List<Subscription> listAllSubscriptions() {
        return Collections.unmodifiableList(subscriptions);
    }

    @Override
    public void addNewSubscription(Subscription subscription) {
        subscriptions.add(subscription);
    }

    @Override
    public void removeSubscription(String topic, String clientID) {
        subscriptions.stream()
            .filter(s -> s.getTopicFilter().toString().equals(topic) && s.getClientId().equals(clientID))
            .findFirst()
            .ifPresent(subscriptions::remove);
    }
}

//session信息的保存注册
public class MemoryQueueRepository implements IQueueRepository {

    private Map<String, Queue<SessionRegistry.EnqueuedMessage>> queues = new HashMap<>();

    @Override
    public Queue<SessionRegistry.EnqueuedMessage> createQueue(String cli, boolean clean) {
        final ConcurrentLinkedQueue<SessionRegistry.EnqueuedMessage> queue = new ConcurrentLinkedQueue<>();
        queues.put(cli, queue);
        return queue;
    }

    @Override
    public Map<String, Queue<SessionRegistry.EnqueuedMessage>> listAllQueues() {
        return Collections.unmodifiableMap(queues);
    }
}

final class MemoryRetainedRepository implements IRetainedRepository {
	//使用hashmap保存
    private final ConcurrentMap<Topic, RetainedMessage> storage = new ConcurrentHashMap<>();

    @Override
    public void cleanRetained(Topic topic) {
        storage.remove(topic);
    }

    @Override
    public void retain(Topic topic, MqttPublishMessage msg) {
        final ByteBuf payload = msg.content();
        byte[] rawPayload = new byte[payload.readableBytes()];
        payload.getBytes(0, rawPayload);
        final RetainedMessage toStore = new RetainedMessage(topic, msg.fixedHeader().qosLevel(), rawPayload);
        storage.put(topic, toStore);
    }

    @Override
    public boolean isEmpty() {
        return storage.isEmpty();
    }

    @Override
    public List<RetainedMessage> retainedOnTopic(String topic) {
        final Topic searchTopic = new Topic(topic);
        final List<RetainedMessage> matchingMessages = new ArrayList<>();
        for (Map.Entry<Topic, RetainedMessage> entry : storage.entrySet()) {
            final Topic scanTopic = entry.getKey();
            if (scanTopic.match(searchTopic)) {
                matchingMessages.add(entry.getValue());
            }
        }
        return matchingMessages;
    }
}

h2内存数据库保存

创建h2数据库

public class H2Builder {

    private static final Logger LOG = LoggerFactory.getLogger(H2Builder.class);

    private final String storePath;//存储路径
    private final int autosaveInterval; // in seconds
    private final ScheduledExecutorService scheduler;//定时计划任务
    /*
    支持开启一个命名的MVMap，支持命名map的rename。
支持 commit与rollback操作。
回滚操作不仅支持MVMap的put动作 还支持从MVStore 中remove map的操作等。但是remove map的操作进行rollback时 仅仅能将map rollback回来，map中的数据不能回滚回来了。 支持命名map的rename的动作回滚。
通过MVMap的isVolatile标识，支持对管理的MVMap可选持久化。
支持压缩与高压缩比压缩。
通过MVMap的能力打开指定版本的数据。
支持文件存储，支持堆外存储。
支持commit动作的延时时间设置。
支持存储加密。
     */
    private MVStore mvStore;

    //建立h2数据库
    public H2Builder(IConfig props, ScheduledExecutorService scheduler) {
        this.storePath = props.getProperty(BrokerConstants.PERSISTENT_STORE_PROPERTY_NAME, "");
        final String autosaveProp = props.getProperty(BrokerConstants.AUTOSAVE_INTERVAL_PROPERTY_NAME, "30");
        this.autosaveInterval = Integer.parseInt(autosaveProp);
        this.scheduler = scheduler;
    }

//初始化h2数据库信息
    @SuppressWarnings("FutureReturnValueIgnored")
    public H2Builder initStore() {
        LOG.info("Initializing H2 store");
        if (storePath == null || storePath.isEmpty()) {
            throw new IllegalArgumentException("H2 store path can't be null or empty");
        }
        mvStore = new MVStore.Builder()
            .fileName(storePath)
            .autoCommitDisabled()//取消自动提交
            .open();//打开存储

        LOG.trace("Scheduling H2 commit task");
        scheduler.scheduleWithFixedDelay(() -> {
            LOG.trace("Committing to H2");
            mvStore.commit();
        }, autosaveInterval, autosaveInterval, TimeUnit.SECONDS);
        return this;
    }

    public ISubscriptionsRepository subscriptionsRepository() {
        return new H2SubscriptionsRepository(mvStore);
    }

    public void closeStore() {
        mvStore.close();
    }

    public IQueueRepository queueRepository() {
        return new H2QueueRepository(mvStore);
    }

    public IRetainedRepository retainedRepository() {
        return new H2RetainedRepository(mvStore);
    }

再细看其中的实现方法

public class H2SubscriptionsRepository implements ISubscriptionsRepository {

    private static final Logger LOG = LoggerFactory.getLogger(H2SubscriptionsRepository.class);
    private static final String SUBSCRIPTIONS_MAP = "subscriptions";

    private MVMap<String, Subscription> subscriptions;

//通过openMap打开获取MVMap表信息
    H2SubscriptionsRepository(MVStore mvStore) {
        this.subscriptions = mvStore.openMap(SUBSCRIPTIONS_MAP);
    }

    /*
    找到所有的订阅主题
     */
    @Override
    public List<Subscription> listAllSubscriptions() {
        LOG.debug("Retrieving existing subscriptions");

        List<Subscription> results = new ArrayList<>();
        //迭代循环获取订阅的主题信息
        Cursor<String, Subscription> mapCursor = subscriptions.cursor(null);
        while (mapCursor.hasNext()) {
            String subscriptionStr = mapCursor.next();
            results.add(mapCursor.getValue());
        }
        LOG.debug("Loaded {} subscriptions", results.size());
        return results;
    }

    /*
    添加主题，根据主题的topic和clientId进行保存
     */
    @Override
    public void addNewSubscription(Subscription subscription) {
        subscriptions.put(subscription.getTopicFilter() + "-" + subscription.getClientId(), subscription);
    }

    /*
    移除主题订阅
     */
    @Override
    public void removeSubscription(String topicFilter, String clientID) {
        subscriptions.remove(topicFilter + "-" + clientID);
    }
}


public class H2QueueRepository implements IQueueRepository {

    private MVStore mvStore;

    public H2QueueRepository(MVStore mvStore) {
        this.mvStore = mvStore;
    }

    @Override
    public Queue<EnqueuedMessage> createQueue(String cli, boolean clean) {
        if (!clean) {
            return new H2PersistentQueue<>(mvStore, cli);
        }
        return new ConcurrentLinkedQueue<>();
    }

    @Override
    public Map<String, Queue<EnqueuedMessage>> listAllQueues() {
        Map<String, Queue<EnqueuedMessage>> result = new HashMap<>();
        mvStore.getMapNames().stream()
            .filter(name -> name.startsWith("queue_") && !name.endsWith("_meta"))
            .map(name -> name.substring("queue_".length()))
            .forEach(name -> result.put(name, new H2PersistentQueue<EnqueuedMessage>(mvStore, name)));
        return result;
    }
}


public class H2RetainedRepository implements IRetainedRepository {

    private final MVMap<Topic, RetainedMessage> queueMap;

    public H2RetainedRepository(MVStore mvStore) {
        this.queueMap = mvStore.openMap("retained_store");
    }

    @Override
    public void cleanRetained(Topic topic) {
        queueMap.remove(topic);
    }

    @Override
    public void retain(Topic topic, MqttPublishMessage msg) {
        final ByteBuf payload = msg.content();
        byte[] rawPayload = new byte[payload.readableBytes()];
        payload.getBytes(0, rawPayload);
        final RetainedMessage toStore = new RetainedMessage(topic, msg.fixedHeader().qosLevel(), rawPayload);
        queueMap.put(topic, toStore);
    }

    @Override
    public boolean isEmpty() {
        return queueMap.isEmpty();
    }

    @Override
    public List<RetainedMessage> retainedOnTopic(String topic) {
        final Topic searchTopic = new Topic(topic);
        final List<RetainedMessage> matchingMessages = new ArrayList<>();
        for (Map.Entry<Topic, RetainedMessage> entry : queueMap.entrySet()) {
            final Topic scanTopic = entry.getKey();
            if (scanTopic.match(searchTopic)) {
                matchingMessages.add(entry.getValue());
            }
        }

        return matchingMessages;
    }
}

以上就是信息的存储方式

初始其他信息

 /*
        初始化订阅的主题，这里使用了像是二叉树的结构
         */
        ISubscriptionsDirectory subscriptions = new CTrieSubscriptionDirectory();
        subscriptions.init(subscriptionsRepository);

        /*
        判断是否需要认证
         */
        final Authorizator authorizator = new Authorizator(authorizatorPolicy);

	//获取订阅主题的session信息
        sessions = new SessionRegistry(subscriptions, queueRepository, authorizator);
		//主要的逻辑处理程序
        dispatcher = new PostOffice(subscriptions, retainedRepository, sessions, interceptor, authorizator);
        //配置信息
        final BrokerConfiguration brokerConfig = new BrokerConfiguration(config);

connection工厂方法

class MQTTConnectionFactory {

    private final BrokerConfiguration brokerConfig;
    private final IAuthenticator authenticator;
    private final SessionRegistry sessionRegistry;
    private final PostOffice postOffice;

    MQTTConnectionFactory(BrokerConfiguration brokerConfig, IAuthenticator authenticator,
                          SessionRegistry sessionRegistry, PostOffice postOffice) {
        this.brokerConfig = brokerConfig;
        this.authenticator = authenticator;
        this.sessionRegistry = sessionRegistry;
        this.postOffice = postOffice;
    }

    MQTTConnection create(Channel channel) {
        return new MQTTConnection(channel, brokerConfig, authenticator, sessionRegistry, postOffice);
    }
}

下面看netty框架在mqtt上的应用

netty

netty是一个支持高并发高可用的框架，简单易用，非常好实现。

/*
这个方法主要就是netty的应用了
*/
@Sharable
public class NewNettyMQTTHandler extends ChannelInboundHandlerAdapter {

    private static final Logger LOG = LoggerFactory.getLogger(NewNettyMQTTHandler.class);

    private static final String ATTR_CONNECTION = "connection";
    private static final AttributeKey<Object> ATTR_KEY_CONNECTION = AttributeKey.valueOf(ATTR_CONNECTION);

    private MQTTConnectionFactory connectionFactory;

	//根据工厂方法创建连接信息
    NewNettyMQTTHandler(MQTTConnectionFactory connectionFactory) {
        this.connectionFactory = connectionFactory;
    }

    private static void mqttConnection(Channel channel, MQTTConnection connection) {
        channel.attr(ATTR_KEY_CONNECTION).set(connection);
    }
	//根据channel通道创建MQTT连接信息
    private static MQTTConnection mqttConnection(Channel channel) {
        return (MQTTConnection) channel.attr(ATTR_KEY_CONNECTION).get();
    }

//当出现连接的时候处理方法
    @Override
    public void channelRead(ChannelHandlerContext ctx, Object message) throws Exception {
        MqttMessage msg = NettyUtils.validateMessage(message);
        final MQTTConnection mqttConnection = mqttConnection(ctx.channel());
        try {
            mqttConnection.handleMessage(msg);
        } catch (Throwable ex) {
            //ctx.fireExceptionCaught(ex);
            LOG.error("Error processing protocol message: {}", msg.fixedHeader().messageType(), ex);
            ctx.channel().close().addListener(new ChannelFutureListener() {
                @Override
                public void operationComplete(ChannelFuture future) {
                    LOG.info("Closed client channel due to exception in processing");
                }
            });
        } finally {
            ReferenceCountUtil.release(msg);
        }
    }
//读取完毕的处理方法
    @Override
    public void channelReadComplete(ChannelHandlerContext ctx) throws Exception {
        final MQTTConnection mqttConnection = mqttConnection(ctx.channel());
        mqttConnection.readCompleted();
    }
//刚接触连接的方法
    @Override
    public void channelActive(ChannelHandlerContext ctx) {
        MQTTConnection connection = connectionFactory.create(ctx.channel());
        mqttConnection(ctx.channel(), connection);
    }

    @Override
    public void channelInactive(ChannelHandlerContext ctx) {
        final MQTTConnection mqttConnection = mqttConnection(ctx.channel());
        mqttConnection.handleConnectionLost();
    }

    @Override
    public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
        LOG.error("Unexpected exception while processing MQTT message. Closing Netty channel. CId={}",
                  NettyUtils.clientID(ctx.channel()), cause);
        ctx.close().addListener(CLOSE_ON_FAILURE);
    }

    @Override
    public void channelWritabilityChanged(ChannelHandlerContext ctx) {
//        if (ctx.channel().isWritable()) {
//            m_processor.notifyChannelWritable(ctx.channel());
//        }
        final MQTTConnection mqttConnection = mqttConnection(ctx.channel());
        mqttConnection.writabilityChanged();
        ctx.fireChannelWritabilityChanged();
    }

    @Override
    public void userEventTriggered(ChannelHandlerContext ctx, Object evt) {
        if (evt instanceof InflightResender.ResendNotAckedPublishes) {
            final MQTTConnection mqttConnection = mqttConnection(ctx.channel());
            mqttConnection.resendNotAckedPublishes();
        }
        ctx.fireUserEventTriggered(evt);
    }

}

在这个方法里要注意channelActive

   @Override
    public void channelActive(ChannelHandlerContext ctx) {
        MQTTConnection connection = connectionFactory.create(ctx.channel());
        mqttConnection(ctx.channel(), connection);
    }

这里根据ctx的通道信息实现了MQTTConnection类。
这里的实现很巧妙，可以细细琢磨琢磨。

在channelRead中处理了连接的信息，比如mqtt协议中的connect/publish/subscribe的协议解析都在这个方法里进行处理，那么下面就解析最主要的mqttconnection类

MQTTConnection

这个类中就主要解析了根据不同连接请求作出的处理和回应。

final class MQTTConnection {
    void handleMessage(MqttMessage msg) {
        MqttMessageType messageType = msg.fixedHeader().messageType();
        LOG.debug("Received MQTT message, type: {}", messageType);
        switch (messageType) {
            case CONNECT:
                processConnect((MqttConnectMessage) msg);
                break;
            case SUBSCRIBE:
                processSubscribe((MqttSubscribeMessage) msg);
                break;
            case UNSUBSCRIBE:
                processUnsubscribe((MqttUnsubscribeMessage) msg);
                break;
            case PUBLISH:
                processPublish((MqttPublishMessage) msg);
                break;
            case PUBREC:
                processPubRec(msg);
                break;
            case PUBCOMP:
                processPubComp(msg);
                break;
            case PUBREL:
                processPubRel(msg);
                break;
            case DISCONNECT:
                processDisconnect(msg);
                break;
            case PUBACK:
                processPubAck(msg);
                break;
            case PINGREQ:
                MqttFixedHeader pingHeader = new MqttFixedHeader(MqttMessageType.PINGRESP, false, AT_MOST_ONCE,
                                                                false, 0);
                MqttMessage pingResp = new MqttMessage(pingHeader);
                channel.writeAndFlush(pingResp).addListener(CLOSE_ON_FAILURE);
                break;
            default:
                LOG.error("Unknown MessageType: {}", messageType);
                break;
        }
    }

    private void processPubComp(MqttMessage msg) {
        final int messageID = ((MqttMessageIdVariableHeader) msg.variableHeader()).messageId();
        bindedSession.processPubComp(messageID);
    }

    private void processPubRec(MqttMessage msg) {
        final int messageID = ((MqttMessageIdVariableHeader) msg.variableHeader()).messageId();
        bindedSession.processPubRec(messageID);
    }

    static MqttMessage pubrel(int messageID) {
        MqttFixedHeader pubRelHeader = new MqttFixedHeader(MqttMessageType.PUBREL, false, AT_LEAST_ONCE, false, 0);
        return new MqttMessage(pubRelHeader, from(messageID));
    }

    private void processPubAck(MqttMessage msg) {
        final int messageID = ((MqttMessageIdVariableHeader) msg.variableHeader()).messageId();
        bindedSession.pubAckReceived(messageID);
    }

    /*
    处理连接请求
     */
    void processConnect(MqttConnectMessage msg) {
        MqttConnectPayload payload = msg.payload();
        String clientId = payload.clientIdentifier();//客户端id
        final String username = payload.userName();//客户端username
        LOG.trace("Processing CONNECT message. CId: {} username: {}", clientId, username);

        if (isNotProtocolVersion(msg, MqttVersion.MQTT_3_1) && isNotProtocolVersion(msg, MqttVersion.MQTT_3_1_1)) {
            LOG.warn("MQTT protocol version is not valid. CId: {}", clientId);
            abortConnection(CONNECTION_REFUSED_UNACCEPTABLE_PROTOCOL_VERSION);//断开连接
            return;
        }
        final boolean cleanSession = msg.variableHeader().isCleanSession();
        //如果客户端Id为空，不允许进行连接
        if (clientId == null || clientId.length() == 0) {
            if (!brokerConfig.isAllowZeroByteClientId()) {
                LOG.info("Broker doesn't permit MQTT empty client ID. Username: {}", username);
                abortConnection(CONNECTION_REFUSED_IDENTIFIER_REJECTED);
                return;
            }

            if (!cleanSession) {
                LOG.info("MQTT client ID cannot be empty for persistent session. Username: {}", username);
                abortConnection(CONNECTION_REFUSED_IDENTIFIER_REJECTED);
                return;
            }

            // Generating client id.
            clientId = UUID.randomUUID().toString().replace("-", "");
            LOG.debug("Client has connected with integration generated id: {}, username: {}", clientId, username);
        }
        //进行判断看是否允许登录
        if (!login(msg, clientId)) {
            abortConnection(CONNECTION_REFUSED_BAD_USER_NAME_OR_PASSWORD);
            channel.close().addListener(CLOSE_ON_FAILURE);
            return;
        }

        final SessionRegistry.SessionCreationResult result;
        try {
            LOG.trace("Binding MQTTConnection to session");
            result = sessionRegistry.createOrReopenSession(msg, clientId, this.getUsername());
            result.session.bind(this);
            bindedSession = result.session;
        } catch (SessionCorruptedException scex) {
            LOG.warn("MQTT session for client ID {} cannot be created", clientId);
            abortConnection(CONNECTION_REFUSED_SERVER_UNAVAILABLE);
            return;
        }

        final boolean msgCleanSessionFlag = msg.variableHeader().isCleanSession();
        boolean isSessionAlreadyPresent = !msgCleanSessionFlag && result.alreadyStored;
        final String clientIdUsed = clientId;
        //发送应答回执
        final MqttConnAckMessage ackMessage = MqttMessageBuilders.connAck()
            .returnCode(CONNECTION_ACCEPTED)//允许连接
            .sessionPresent(isSessionAlreadyPresent).build();
        channel.writeAndFlush(ackMessage).addListener(new ChannelFutureListener() {
            @Override
            public void operationComplete(ChannelFuture future) throws Exception {
                if (future.isSuccess()) {
                    LOG.trace("CONNACK sent, channel: {}", channel);
                    if (!result.session.completeConnection()) {
                        // send DISCONNECT and close the channel
                        final MqttMessage disconnectMsg = MqttMessageBuilders.disconnect().build();
                        channel.writeAndFlush(disconnectMsg).addListener(CLOSE);
                        LOG.warn("CONNACK is sent but the session created can't transition in CONNECTED state");
                    } else {
                        NettyUtils.clientID(channel, clientIdUsed);
                        connected = true;
                        // OK continue with sending queued messages and normal flow

                        if (result.mode == SessionRegistry.CreationModeEnum.REOPEN_EXISTING) {
                            result.session.sendQueuedMessagesWhileOffline();
                        }

                        initializeKeepAliveTimeout(channel, msg, clientIdUsed);
                        setupInflightResender(channel);

                        postOffice.dispatchConnection(msg);
                        LOG.trace("dispatch connection: {}", msg.toString());
                    }
                } else {
                    bindedSession.disconnect();
                    sessionRegistry.remove(bindedSession);
                    LOG.error("CONNACK send failed, cleanup session and close the connection", future.cause());
                    channel.close();
                }

            }
        });
    }

    private void setupInflightResender(Channel channel) {
        channel.pipeline()
            .addFirst("inflightResender", new InflightResender(5_000, TimeUnit.MILLISECONDS));
    }

    private void initializeKeepAliveTimeout(Channel channel, MqttConnectMessage msg, String clientId) {
        int keepAlive = msg.variableHeader().keepAliveTimeSeconds();
        NettyUtils.keepAlive(channel, keepAlive);
        NettyUtils.cleanSession(channel, msg.variableHeader().isCleanSession());
        NettyUtils.clientID(channel, clientId);
        int idleTime = Math.round(keepAlive * 1.5f);
        setIdleTime(channel.pipeline(), idleTime);

        LOG.debug("Connection has been configured CId={}, keepAlive={}, removeTemporaryQoS2={}, idleTime={}",
            clientId, keepAlive, msg.variableHeader().isCleanSession(), idleTime);
    }

    private void setIdleTime(ChannelPipeline pipeline, int idleTime) {
        if (pipeline.names().contains("idleStateHandler")) {
            pipeline.remove("idleStateHandler");
        }
        pipeline.addFirst("idleStateHandler", new IdleStateHandler(idleTime, 0, 0));
    }

    private boolean isNotProtocolVersion(MqttConnectMessage msg, MqttVersion version) {
        return msg.variableHeader().version() != version.protocolLevel();
    }

    private void abortConnection(MqttConnectReturnCode returnCode) {
        MqttConnAckMessage badProto = MqttMessageBuilders.connAck()
            .returnCode(returnCode)
            .sessionPresent(false).build();
        channel.writeAndFlush(badProto).addListener(FIRE_EXCEPTION_ON_FAILURE);
        channel.close().addListener(CLOSE_ON_FAILURE);
    }

    private boolean login(MqttConnectMessage msg, final String clientId) {
        // handle user authentication
        if (msg.variableHeader().hasUserName()) {
            byte[] pwd = null;
            if (msg.variableHeader().hasPassword()) {
                pwd = msg.payload().passwordInBytes();
            } else if (!brokerConfig.isAllowAnonymous()) {
                LOG.info("Client didn't supply any password and MQTT anonymous mode is disabled CId={}", clientId);
                return false;
            }
            final String login = msg.payload().userName();
            if (!authenticator.checkValid(clientId, login, pwd)) {
                LOG.info("Authenticator has rejected the MQTT credentials CId={}, username={}", clientId, login);
                return false;
            }
            NettyUtils.userName(channel, login);
        } else if (!brokerConfig.isAllowAnonymous()) {
            //如果客户端没有名称，而且还不允许匿名登录，这里就不让登录了
            LOG.info("Client didn't supply any credentials and MQTT anonymous mode is disabled. CId={}", clientId);
            return false;
        }
        return true;
    }

    void handleConnectionLost() {
        String clientID = NettyUtils.clientID(channel);
        if (clientID == null || clientID.isEmpty()) {
            return;
        }
        LOG.info("Notifying connection lost event");
        if (bindedSession.hasWill()) {
            postOffice.fireWill(bindedSession.getWill());
        }
        if (bindedSession.isClean()) {
            LOG.debug("Remove session for client");
            sessionRegistry.remove(bindedSession);
        } else {
            bindedSession.disconnect();
        }
        connected = false;
        //dispatch connection lost to intercept.
        String userName = NettyUtils.userName(channel);
        postOffice.dispatchConnectionLost(clientID,userName);
        LOG.trace("dispatch disconnection: userName={}", userName);
    }

    boolean isConnected() {
        return connected;
    }

    void dropConnection() {
        channel.close().addListener(FIRE_EXCEPTION_ON_FAILURE);
    }

    void processDisconnect(MqttMessage msg) {
        final String clientID = NettyUtils.clientID(channel);
        LOG.trace("Start DISCONNECT");
        if (!connected) {
            LOG.info("DISCONNECT received on already closed connection");
            return;
        }
        bindedSession.disconnect();
        connected = false;
        channel.close().addListener(FIRE_EXCEPTION_ON_FAILURE);
        LOG.trace("Processed DISCONNECT");
        String userName = NettyUtils.userName(channel);
        postOffice.dispatchDisconnection(clientID, userName);
        LOG.trace("dispatch disconnection userName={}", userName);
    }

    void processSubscribe(MqttSubscribeMessage msg) {
        final String clientID = NettyUtils.clientID(channel);
        if (!connected) {//如果客户端连接不成功
            LOG.warn("SUBSCRIBE received on already closed connection");
            dropConnection();
            return;
        }
        postOffice.subscribeClientToTopics(msg, clientID, NettyUtils.userName(channel), this);
    }

    void sendSubAckMessage(int messageID, MqttSubAckMessage ackMessage) {
        LOG.trace("Sending SUBACK response messageId: {}", messageID);
        channel.writeAndFlush(ackMessage).addListener(FIRE_EXCEPTION_ON_FAILURE);
    }

    private void processUnsubscribe(MqttUnsubscribeMessage msg) {
        List<String> topics = msg.payload().topics();
        String clientID = NettyUtils.clientID(channel);

        LOG.trace("Processing UNSUBSCRIBE message. topics: {}", topics);
        postOffice.unsubscribe(topics, this, msg.variableHeader().messageId());
    }

    void sendUnsubAckMessage(List<String> topics, String clientID, int messageID) {
        MqttFixedHeader fixedHeader = new MqttFixedHeader(MqttMessageType.UNSUBACK, false, AT_MOST_ONCE,
            false, 0);
        MqttUnsubAckMessage ackMessage = new MqttUnsubAckMessage(fixedHeader, from(messageID));

        LOG.trace("Sending UNSUBACK message. messageId: {}, topics: {}", messageID, topics);
        channel.writeAndFlush(ackMessage).addListener(FIRE_EXCEPTION_ON_FAILURE);
        LOG.trace("Client unsubscribed from topics <{}>", topics);
    }

    //客户端发布消息到服务器端
    void processPublish(MqttPublishMessage msg) {
        final MqttQoS qos = msg.fixedHeader().qosLevel();
        final String username = NettyUtils.userName(channel);
        final String topicName = msg.variableHeader().topicName();
        final String clientId = getClientId();
        final int messageID = msg.variableHeader().packetId();
        LOG.trace("Processing PUBLISH message, topic: {}, messageId: {}, qos: {}", topicName, messageID, qos);
        ByteBuf payload = msg.payload();
        final boolean retain = msg.fixedHeader().isRetain();
        final Topic topic = new Topic(topicName);
        if (!topic.isValid()) {
            LOG.debug("Drop connection because of invalid topic format");
            dropConnection();
        }
        switch (qos) {
            case AT_MOST_ONCE:
                postOffice.receivedPublishQos0(topic, username, clientId, payload, retain, msg);
                break;
            case AT_LEAST_ONCE: {
                postOffice.receivedPublishQos1(this, topic, username, payload, messageID, retain, msg);
                break;
            }
            case EXACTLY_ONCE: {
                bindedSession.receivedPublishQos2(messageID, msg);
                postOffice.receivedPublishQos2(this, msg, username);
//                msg.release();
                break;
            }
            default:
                LOG.error("Unknown QoS-Type:{}", qos);
                break;
        }
    }

    void sendPublishReceived(int messageID) {
        LOG.trace("sendPubRec invoked, messageID: {}", messageID);
        MqttFixedHeader fixedHeader = new MqttFixedHeader(MqttMessageType.PUBREC, false, AT_MOST_ONCE,
            false, 0);
        MqttPubAckMessage pubRecMessage = new MqttPubAckMessage(fixedHeader, from(messageID));
        sendIfWritableElseDrop(pubRecMessage);
    }

    private void processPubRel(MqttMessage msg) {
        final int messageID = ((MqttMessageIdVariableHeader) msg.variableHeader()).messageId();
        bindedSession.receivedPubRelQos2(messageID);
        sendPubCompMessage(messageID);
    }

    //发送消息给订阅方
    void sendPublish(MqttPublishMessage publishMsg) {
        final int packetId = publishMsg.variableHeader().packetId();
        final String topicName = publishMsg.variableHeader().topicName();
        final String clientId = getClientId();
        MqttQoS qos = publishMsg.fixedHeader().qosLevel();
        if (LOG.isTraceEnabled()) {
            LOG.trace("Sending PUBLISH({}) message. MessageId={}, topic={}, payload={}", qos, packetId, topicName,
                DebugUtils.payload2Str(publishMsg.payload()));
        } else {
            LOG.debug("Sending PUBLISH({}) message. MessageId={}, topic={}", qos, packetId, topicName);
        }
        sendIfWritableElseDrop(publishMsg);
    }

    void sendIfWritableElseDrop(MqttMessage msg) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("OUT {}", msg.fixedHeader().messageType());
        }
        if (channel.isWritable()) {
            ChannelFuture channelFuture;
            if (brokerConfig.isImmediateBufferFlush()) {
                channelFuture = channel.writeAndFlush(msg);
            } else {
                channelFuture = channel.write(msg);
            }
            channelFuture.addListener(FIRE_EXCEPTION_ON_FAILURE);
        }
    }

    public void writabilityChanged() {
        if (channel.isWritable()) {
            LOG.debug("Channel is again writable");
            bindedSession.writabilityChanged();
        }
    }

    void sendPubAck(int messageID) {
        LOG.trace("sendPubAck invoked");
        MqttFixedHeader fixedHeader = new MqttFixedHeader(MqttMessageType.PUBACK, false, AT_MOST_ONCE,
                                                  false, 0);
        MqttPubAckMessage pubAckMessage = new MqttPubAckMessage(fixedHeader, from(messageID));
        sendIfWritableElseDrop(pubAckMessage);
    }

    private void sendPubCompMessage(int messageID) {
        LOG.trace("Sending PUBCOMP message messageId: {}", messageID);
        MqttFixedHeader fixedHeader = new MqttFixedHeader(MqttMessageType.PUBCOMP, false, AT_MOST_ONCE, false, 0);
        MqttMessage pubCompMessage = new MqttMessage(fixedHeader, from(messageID));
        sendIfWritableElseDrop(pubCompMessage);
    }

    String getClientId() {
        return NettyUtils.clientID(channel);
    }

    String getUsername() {
        return NettyUtils.userName(channel);
    }

    //发送消息给订阅方
    public void sendPublishRetainedQos0(Topic topic, MqttQoS qos, ByteBuf payload) {
        MqttPublishMessage publishMsg = retainedPublish(topic.toString(), qos, payload);
        sendPublish(publishMsg);
    }

    public void sendPublishRetainedWithPacketId(Topic topic, MqttQoS qos, ByteBuf payload) {
        final int packetId = nextPacketId();
        MqttPublishMessage publishMsg = retainedPublishWithMessageId(topic.toString(), qos, payload, packetId);
        sendPublish(publishMsg);
    }

    private static MqttPublishMessage retainedPublish(String topic, MqttQoS qos, ByteBuf message) {
        return retainedPublishWithMessageId(topic, qos, message, 0);
    }

    //组织订阅方的消息
    //topic主题
    //qos消息到达类型
    //message消息
    //messageId消息id
    private static MqttPublishMessage retainedPublishWithMessageId(String topic, MqttQoS qos, ByteBuf message,
                                                                   int messageId) {
        MqttFixedHeader fixedHeader = new MqttFixedHeader(MqttMessageType.PUBLISH, false, qos, true, 0);
        MqttPublishVariableHeader varHeader = new MqttPublishVariableHeader(topic, messageId);
        return new MqttPublishMessage(fixedHeader, varHeader, message);
    }

    // TODO move this method in Session
    void sendPublishNotRetainedQos0(Topic topic, MqttQoS qos, ByteBuf payload) {
        MqttPublishMessage publishMsg = notRetainedPublish(topic.toString(), qos, payload);
        sendPublish(publishMsg);
    }

    static MqttPublishMessage notRetainedPublish(String topic, MqttQoS qos, ByteBuf message) {
        return notRetainedPublishWithMessageId(topic, qos, message, 0);
    }

    static MqttPublishMessage notRetainedPublishWithMessageId(String topic, MqttQoS qos, ByteBuf message,
                                                              int messageId) {
        MqttFixedHeader fixedHeader = new MqttFixedHeader(MqttMessageType.PUBLISH, false, qos, false, 0);
        MqttPublishVariableHeader varHeader = new MqttPublishVariableHeader(topic, messageId);
        return new MqttPublishMessage(fixedHeader, varHeader, message);
    }

    public void resendNotAckedPublishes() {
        bindedSession.resendInflightNotAcked();
    }

    int nextPacketId() {
        return lastPacketId.incrementAndGet();
    }

    @Override
    public String toString() {
        return "MQTTConnection{channel=" + channel + ", connected=" + connected + '}';
    }

    InetSocketAddress remoteAddress() {
        return (InetSocketAddress) channel.remoteAddress();
    }

    public void readCompleted() {
        LOG.debug("readCompleted client CId: {}", getClientId());
        if (getClientId() != null) {
            // TODO drain all messages in target's session in-flight message queue
            bindedSession.flushAllQueuedMessages();
        }
    }
}

下面以connect 、publish和subscribe三个连接状态进行分析

connect

/*
    处理连接请求
     */
    void processConnect(MqttConnectMessage msg) {
        MqttConnectPayload payload = msg.payload();
        String clientId = payload.clientIdentifier();//客户端id
        final String username = payload.userName();//客户端username
        LOG.trace("Processing CONNECT message. CId: {} username: {}", clientId, username);

        if (isNotProtocolVersion(msg, MqttVersion.MQTT_3_1) && isNotProtocolVersion(msg, MqttVersion.MQTT_3_1_1)) {
            LOG.warn("MQTT protocol version is not valid. CId: {}", clientId);
            abortConnection(CONNECTION_REFUSED_UNACCEPTABLE_PROTOCOL_VERSION);//断开连接
            return;
        }
        final boolean cleanSession = msg.variableHeader().isCleanSession();
        //如果客户端Id为空，不允许进行连接
        if (clientId == null || clientId.length() == 0) {
            if (!brokerConfig.isAllowZeroByteClientId()) {
                LOG.info("Broker doesn't permit MQTT empty client ID. Username: {}", username);
                abortConnection(CONNECTION_REFUSED_IDENTIFIER_REJECTED);
                return;
            }

            if (!cleanSession) {
                LOG.info("MQTT client ID cannot be empty for persistent session. Username: {}", username);
                abortConnection(CONNECTION_REFUSED_IDENTIFIER_REJECTED);
                return;
            }

            // Generating client id.
            clientId = UUID.randomUUID().toString().replace("-", "");
            LOG.debug("Client has connected with integration generated id: {}, username: {}", clientId, username);
        }
        //进行判断看是否允许登录
        if (!login(msg, clientId)) {
            abortConnection(CONNECTION_REFUSED_BAD_USER_NAME_OR_PASSWORD);
            channel.close().addListener(CLOSE_ON_FAILURE);
            return;
        }

        final SessionRegistry.SessionCreationResult result;
        try {
            LOG.trace("Binding MQTTConnection to session");
            result = sessionRegistry.createOrReopenSession(msg, clientId, this.getUsername());
            result.session.bind(this);
            bindedSession = result.session;
        } catch (SessionCorruptedException scex) {
            LOG.warn("MQTT session for client ID {} cannot be created", clientId);
            abortConnection(CONNECTION_REFUSED_SERVER_UNAVAILABLE);
            return;
        }

        final boolean msgCleanSessionFlag = msg.variableHeader().isCleanSession();
        boolean isSessionAlreadyPresent = !msgCleanSessionFlag && result.alreadyStored;
        final String clientIdUsed = clientId;
        //发送应答回执
        final MqttConnAckMessage ackMessage = MqttMessageBuilders.connAck()
            .returnCode(CONNECTION_ACCEPTED)//允许连接
            .sessionPresent(isSessionAlreadyPresent).build();
        channel.writeAndFlush(ackMessage).addListener(new ChannelFutureListener() {
            @Override
            public void operationComplete(ChannelFuture future) throws Exception {
                if (future.isSuccess()) {
                    LOG.trace("CONNACK sent, channel: {}", channel);
                    if (!result.session.completeConnection()) {
                        // send DISCONNECT and close the channel
                        final MqttMessage disconnectMsg = MqttMessageBuilders.disconnect().build();
                        channel.writeAndFlush(disconnectMsg).addListener(CLOSE);
                        LOG.warn("CONNACK is sent but the session created can't transition in CONNECTED state");
                    } else {
                        NettyUtils.clientID(channel, clientIdUsed);
                        connected = true;
                        // OK continue with sending queued messages and normal flow

                        if (result.mode == SessionRegistry.CreationModeEnum.REOPEN_EXISTING) {
                            result.session.sendQueuedMessagesWhileOffline();
                        }

                        initializeKeepAliveTimeout(channel, msg, clientIdUsed);
                        setupInflightResender(channel);

                        postOffice.dispatchConnection(msg);
                        LOG.trace("dispatch connection: {}", msg.toString());
                    }
                } else {
                    bindedSession.disconnect();
                    sessionRegistry.remove(bindedSession);
                    LOG.error("CONNACK send failed, cleanup session and close the connection", future.cause());
                    channel.close();
                }

            }
        });
    }

主要看这三个方法

result = sessionRegistry.createOrReopenSession(msg, clientId, this.getUsername());
            result.session.bind(this);
            bindedSession = result.session;



    SessionCreationResult createOrReopenSession(MqttConnectMessage msg, String clientId, String username) {
        SessionCreationResult postConnectAction;
        //创建session连接信息，包含clientid,sessionclean等一系列信息
        final Session newSession = createNewSession(msg, clientId);
        final Session oldSession = pool.get(clientId);
        if (oldSession == null) {
            // case 1
            postConnectAction = new SessionCreationResult(newSession, CreationModeEnum.CREATED_CLEAN_NEW, false);

            // publish the session
            final Session previous = pool.putIfAbsent(clientId, newSession);
            final boolean success = previous == null;

            if (success) {
                LOG.trace("case 1, not existing session with CId {}", clientId);
            } else {
                postConnectAction = reopenExistingSession(msg, clientId, previous, newSession, username);
            }
        } else {
            postConnectAction = reopenExistingSession(msg, clientId, oldSession, newSession, username);
        }
        return postConnectAction;
    }

创建session成功后，需要发布连接成功回执MqttConnAckMessage

//发送应答回执
        final MqttConnAckMessage ackMessage = MqttMessageBuilders.connAck()
            .returnCode(CONNECTION_ACCEPTED)//允许连接
            .sessionPresent(isSessionAlreadyPresent).build();
            //在这里增加了很多成功执行后的处理
        channel.writeAndFlush(ackMessage).addListener(new ChannelFutureListener() {
            @Override
            public void operationComplete(ChannelFuture future) throws Exception {
                if (future.isSuccess()) {
                    LOG.trace("CONNACK sent, channel: {}", channel);
                    if (!result.session.completeConnection()) {
                        // send DISCONNECT and close the channel
                        final MqttMessage disconnectMsg = MqttMessageBuilders.disconnect().build();
                        channel.writeAndFlush(disconnectMsg).addListener(CLOSE);
                        LOG.warn("CONNACK is sent but the session created can't transition in CONNECTED state");
                    } else {
                        NettyUtils.clientID(channel, clientIdUsed);
                        connected = true;
                        // OK continue with sending queued messages and normal flow

                        if (result.mode == SessionRegistry.CreationModeEnum.REOPEN_EXISTING) {
                            result.session.sendQueuedMessagesWhileOffline();
                        }

                        initializeKeepAliveTimeout(channel, msg, clientIdUsed);
                        setupInflightResender(channel);

                        postOffice.dispatchConnection(msg);
                        LOG.trace("dispatch connection: {}", msg.toString());
                    }
                } else {
                    bindedSession.disconnect();
                    sessionRegistry.remove(bindedSession);
                    LOG.error("CONNACK send failed, cleanup session and close the connection", future.cause());
                    channel.close();
                }

            }
        });

这样整个连接就处理完了

publish发布主题

 //客户端发布消息到服务器端
    void processPublish(MqttPublishMessage msg) {
        final MqttQoS qos = msg.fixedHeader().qosLevel();
        final String username = NettyUtils.userName(channel);
        final String topicName = msg.variableHeader().topicName();
        final String clientId = getClientId();
        final int messageID = msg.variableHeader().packetId();
        LOG.trace("Processing PUBLISH message, topic: {}, messageId: {}, qos: {}", topicName, messageID, qos);
        ByteBuf payload = msg.payload();
        final boolean retain = msg.fixedHeader().isRetain();
        final Topic topic = new Topic(topicName);
        if (!topic.isValid()) {
            LOG.debug("Drop connection because of invalid topic format");
            dropConnection();
        }
        switch (qos) {
            case AT_MOST_ONCE://至多一次
                postOffice.receivedPublishQos0(topic, username, clientId, payload, retain, msg);
                break;
            case AT_LEAST_ONCE: {//至少一次
                postOffice.receivedPublishQos1(this, topic, username, payload, messageID, retain, msg);
                break;
            }
            case EXACTLY_ONCE: {//只有一次
                bindedSession.receivedPublishQos2(messageID, msg);
                postOffice.receivedPublishQos2(this, msg, username);
//                msg.release();
                break;
            }
            default:
                LOG.error("Unknown QoS-Type:{}", qos);
                break;
        }
    }

其实我们只要关注AT_MOST_ONCE和AT_LEAST_ONCE就行了，AT_LEAST_ONCE一般用的很少
这个postOffice类就是初始时创建好的，用于处理主题的订阅和发布的。

//topic 主题
//clinetId 客户端Id
//payload 消息体内容
//msg 消息
    void receivedPublishQos0(Topic topic, String username, String clientID, ByteBuf payload, boolean retain,
                             MqttPublishMessage msg) {
        if (!authorizator.canWrite(topic, username, clientID)) {
            LOG.error("client is not authorized to publish on topic: {}", topic);
            return;
        }
        publish2Subscribers(payload, topic, AT_MOST_ONCE);

        if (retain) {
            // QoS == 0 && retain => clean old retained
            retainedRepository.cleanRetained(topic);
        }

        interceptor.notifyTopicPublished(msg, clientID, username);
    }

//
    private void publish2Subscribers(ByteBuf origPayload, Topic topic, MqttQoS publishingQos) {
        //查看是否有订阅这个主题的消息
        Set<Subscription> topicMatchingSubscriptions = subscriptions.matchQosSharpening(topic);

        for (final Subscription sub : topicMatchingSubscriptions) {
        //比较两个的消息等级，按照最低的处理
            MqttQoS qos = lowerQosToTheSubscriptionDesired(sub, publishingQos);
            //获取订阅主题的session信息，根据sub的客户端id来配置
            Session targetSession = this.sessionRegistry.retrieve(sub.getClientId());
			//判断session是否存在
            boolean isSessionPresent = targetSession != null;
            if (isSessionPresent) {
            //订阅主题的客户端还在线
                LOG.debug("Sending PUBLISH message to active subscriber CId: {}, topicFilter: {}, qos: {}",
                          sub.getClientId(), sub.getTopicFilter(), qos);
                // we need to retain because duplicate only copy r/w indexes and don't retain() causing refCnt = 0
                ByteBuf payload = origPayload.retainedDuplicate();
                //如果session存在，就发送消息
                targetSession.sendPublishOnSessionAtQos(topic, qos, payload);
            } else {
                // If we are, the subscriber disconnected after the subscriptions tree selected that session as a
                // destination.
                LOG.debug("PUBLISH to not yet present session. CId: {}, topicFilter: {}, qos: {}", sub.getClientId(),
                          sub.getTopicFilter(), qos);
            }
        }
    }
//再看看这个方法
//sendPublishOnSessionAtQos
//这里面有用到了mqttconnection了，一会画个类图来关注下这几个类之间的关系
public void sendPublishOnSessionAtQos(Topic topic, MqttQoS qos, ByteBuf payload) {
        switch (qos) {
            case AT_MOST_ONCE:
                if (connected()) {
                    mqttConnection.sendPublishNotRetainedQos0(topic, qos, payload);
                }
                break;
            case AT_LEAST_ONCE:
                sendPublishQos1(topic, qos, payload);
                break;
            case EXACTLY_ONCE:
                sendPublishQos2(topic, qos, payload);
                break;
            case FAILURE:
                LOG.error("Not admissible");
        }
    }
	//最终的发送方法
    //发送消息给订阅方
    void sendPublish(MqttPublishMessage publishMsg) {
        final int packetId = publishMsg.variableHeader().packetId();
        final String topicName = publishMsg.variableHeader().topicName();
        final String clientId = getClientId();
        MqttQoS qos = publishMsg.fixedHeader().qosLevel();
        if (LOG.isTraceEnabled()) {
            LOG.trace("Sending PUBLISH({}) message. MessageId={}, topic={}, payload={}", qos, packetId, topicName,
                DebugUtils.payload2Str(publishMsg.payload()));
        } else {
            LOG.debug("Sending PUBLISH({}) message. MessageId={}, topic={}", qos, packetId, topicName);
        }
        sendIfWritableElseDrop(publishMsg);
    }

    void sendIfWritableElseDrop(MqttMessage msg) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("OUT {}", msg.fixedHeader().messageType());
        }
        if (channel.isWritable()) {
            ChannelFuture channelFuture;
            if (brokerConfig.isImmediateBufferFlush()) {
                channelFuture = channel.writeAndFlush(msg);
            } else {
                channelFuture = channel.write(msg);
            }
            channelFuture.addListener(FIRE_EXCEPTION_ON_FAILURE);
        }
    }

再来看下AT_LEAST_ONCE这个处理

    void receivedPublishQos1(MQTTConnection connection, Topic topic, String username, ByteBuf payload, int messageID,
                             boolean retain, MqttPublishMessage msg) {
        // verify if topic can be write
        topic.getTokens();
        if (!topic.isValid()) {
            LOG.warn("Invalid topic format, force close the connection");
            connection.dropConnection();
            return;
        }
        final String clientId = connection.getClientId();
        if (!authorizator.canWrite(topic, username, clientId)) {
            LOG.error("MQTT client: {} is not authorized to publish on topic: {}", clientId, topic);
            return;
        }

        publish2Subscribers(payload, topic, AT_LEAST_ONCE);

        connection.sendPubAck(messageID);

        if (retain) {
            if (!payload.isReadable()) {
                retainedRepository.cleanRetained(topic);
            } else {
                // before wasn't stored
                retainedRepository.retain(topic, msg);
            }
        }
        interceptor.notifyTopicPublished(msg, clientId, username);
    }


    private void publish2Subscribers(ByteBuf origPayload, Topic topic, MqttQoS publishingQos) {
        //查看是否有订阅这个主题的消息
        Set<Subscription> topicMatchingSubscriptions = subscriptions.matchQosSharpening(topic);

        for (final Subscription sub : topicMatchingSubscriptions) {
            MqttQoS qos = lowerQosToTheSubscriptionDesired(sub, publishingQos);
            Session targetSession = this.sessionRegistry.retrieve(sub.getClientId());

            boolean isSessionPresent = targetSession != null;
            if (isSessionPresent) {
                LOG.debug("Sending PUBLISH message to active subscriber CId: {}, topicFilter: {}, qos: {}",
                          sub.getClientId(), sub.getTopicFilter(), qos);
                // we need to retain because duplicate only copy r/w indexes and don't retain() causing refCnt = 0
                ByteBuf payload = origPayload.retainedDuplicate();
                targetSession.sendPublishOnSessionAtQos(topic, qos, payload);
            } else {
                // If we are, the subscriber disconnected after the subscriptions tree selected that session as a
                // destination.
                LOG.debug("PUBLISH to not yet present session. CId: {}, topicFilter: {}, qos: {}", sub.getClientId(),
                          sub.getTopicFilter(), qos);
            }
        }
    }



    private void sendPublishQos1(Topic topic, MqttQoS qos, ByteBuf payload) {
        if (!connected() && isClean()) {
            //pushing messages to disconnected not clean session
            return;
        }

        if (canSkipQueue()) {
            inflightSlots.decrementAndGet();
            int packetId = mqttConnection.nextPacketId();
            inflightWindow.put(packetId, new SessionRegistry.PublishedMessage(topic, qos, payload));
            inflightTimeouts.add(new InFlightPacket(packetId, FLIGHT_BEFORE_RESEND_MS));
            MqttPublishMessage publishMsg = MQTTConnection.notRetainedPublishWithMessageId(topic.toString(), qos,
                                                                                           payload, packetId);
            mqttConnection.sendPublish(publishMsg);

            // TODO drainQueueToConnection();?
        } else {
            final SessionRegistry.PublishedMessage msg = new SessionRegistry.PublishedMessage(topic, qos, payload);
            sessionQueue.add(msg);
        }
    }

subscribe订阅

    void processSubscribe(MqttSubscribeMessage msg) {
        final String clientID = NettyUtils.clientID(channel);
        if (!connected) {//如果客户端连接不成功
            LOG.warn("SUBSCRIBE received on already closed connection");
            dropConnection();
            return;
        }
        postOffice.subscribeClientToTopics(msg, clientID, NettyUtils.userName(channel), this);
    }

    /*
    订阅客户端的主题
     */
    public void subscribeClientToTopics(MqttSubscribeMessage msg, String clientID, String username,
                                        MQTTConnection mqttConnection) {
        // verify which topics of the subscribe ongoing has read access permission
        int messageID = messageId(msg);
        //判断是否允许订阅这个主题
        List<MqttTopicSubscription> ackTopics = authorizator.verifyTopicsReadAccess(clientID, username, msg);
        //订阅主题回执消息类型
        MqttSubAckMessage ackMessage = doAckMessageFromValidateFilters(ackTopics, messageID);

        // store topics subscriptions in session
        List<Subscription> newSubscriptions = ackTopics.stream()
            .filter(req -> req.qualityOfService() != FAILURE)//过滤掉失败的消息主题
            .map(req -> {
                final Topic topic = new Topic(req.topicName());
                return new Subscription(clientID, topic, req.qualityOfService());
            }).collect(Collectors.toList());

        for (Subscription subscription : newSubscriptions) {
            subscriptions.add(subscription);
        }

        // add the subscriptions to Session
        Session session = sessionRegistry.retrieve(clientID);
        session.addSubscriptions(newSubscriptions);

        // send ack message
        //发送应答回执消息
        mqttConnection.sendSubAckMessage(messageID, ackMessage);

        publishRetainedMessagesForSubscriptions(clientID, newSubscriptions);

        for (Subscription subscription : newSubscriptions) {
            interceptor.notifyTopicSubscribed(subscription, username);
        }
    }


    private void publishRetainedMessagesForSubscriptions(String clientID, List<Subscription> newSubscriptions) {
        Session targetSession = this.sessionRegistry.retrieve(clientID);
        for (Subscription subscription : newSubscriptions) {
            final String topicFilter = subscription.getTopicFilter().toString();
            final List<RetainedMessage> retainedMsgs = retainedRepository.retainedOnTopic(topicFilter);

            if (retainedMsgs.isEmpty()) {
                // not found
                continue;
            }
            for (RetainedMessage retainedMsg : retainedMsgs) {
                final MqttQoS retainedQos = retainedMsg.qosLevel();
                MqttQoS qos = lowerQosToTheSubscriptionDesired(subscription, retainedQos);

                final ByteBuf payloadBuf = Unpooled.wrappedBuffer(retainedMsg.getPayload());
                //发送消息给订阅方
                targetSession.sendRetainedPublishOnSessionAtQos(retainedMsg.getTopic(), qos, payloadBuf);
            }
        }
    }

类图关系

下面主要看看这及各类之间的关系

总结

可以根据类图再仔细研究下其中的几个类