1 processRequest方法
1.1 processRequest方法的整体流程
上文中我们学习了关于PullMessageProcessor处理器的原理，这一篇开始探究其源码的构成，拉消息 的主体逻辑如下：
/**
* @param channel 服务器与客户端netty通道
* @param request 客户端请求（RemotingCommand）
* @param brokerAllowSuspend 是否允许服务器端长轮询
*/
private RemotingCommand processRequest(final Channel channel, RemotingCommand request, boolean brokerAllowSuspend)
throws RemotingCommandException {
// 创建服务器端对本请求的“响应对象”，注意：header 类型 PullMessageResponseHeader
RemotingCommand response = RemotingCommand.createResponseCommand(PullMessageResponseHeader.class);
// 获取response对象的 header
final PullMessageResponseHeader responseHeader = (PullMessageResponseHeader) response.readCustomHeader();
// 从request请求中，解析出“requestHeader”对象，类型是：PullMessageRequestHeader
final PullMessageRequestHeader requestHeader =
(PullMessageRequestHeader) request.decodeCommandCustomHeader(PullMessageRequestHeader.class);
// 设置响应对象的opaque 为 request请求的 opaque，为什么要设置？
// 客户端需要根据 response opaque 找到 ResponseFuture 然后完成后面的事情。
response.setOpaque(request.getOpaque());
log.debug("receive PullMessage request command, {}", request);
//接下来的代码就是各种校验
/**
* 校验服务器端是否可读
*/
if (!PermName.isReadable(this.brokerController.getBrokerConfig().getBrokerPermission())) {
response.setCode(ResponseCode.NO_PERMISSION);
response.setRemark(String.format("the broker[%s] pulling message is forbidden", this.brokerController.getBrokerConfig().getBrokerIP1()));
return response;
}
/**
* 获取消费者组配置信息，如果为 null，则也是直接返回
*/
SubscriptionGroupConfig subscriptionGroupConfig =
this.brokerController.getSubscriptionGroupManager().findSubscriptionGroupConfig(requestHeader.getConsumerGroup());
if (null == subscriptionGroupConfig) {
response.setCode(ResponseCode.SUBSCRIPTION_GROUP_NOT_EXIST);
response.setRemark(String.format("subscription group [%s] does not exist, %s", requestHeader.getConsumerGroup(), FAQUrl.suggestTodo(FAQUrl.SUBSCRIPTION_GROUP_NOT_EXIST)));
return response;
}
if (!subscriptionGroupConfig.isConsumeEnable()) {
response.setCode(ResponseCode.NO_PERMISSION);
response.setRemark("subscription group no permission, " + requestHeader.getConsumerGroup());
return response;
}
// 是否允许服务器端长轮询（true）
final boolean hasSuspendFlag = PullSysFlag.hasSuspendFlag(requestHeader.getSysFlag());
// 客户端是否提交 offset（true）
final boolean hasCommitOffsetFlag = PullSysFlag.hasCommitOffsetFlag(requestHeader.getSysFlag());
// 客户端请求是否包含 订阅数据(false)
final boolean hasSubscriptionFlag = PullSysFlag.hasSubscriptionFlag(requestHeader.getSysFlag());
//长轮询允许的时间长度 15秒
final long suspendTimeoutMillisLong = hasSuspendFlag ? requestHeader.getSuspendTimeoutMillis() : 0;
//每个主题都会创建TopicConfig对象
TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(requestHeader.getTopic());
if (null == topicConfig) {
log.error("the topic {} not exist, consumer: {}", requestHeader.getTopic(), RemotingHelper.parseChannelRemoteAddr(channel));
response.setCode(ResponseCode.TOPIC_NOT_EXIST);
response.setRemark(String.format("topic[%s] not exist, apply first please! %s", requestHeader.getTopic(), FAQUrl.suggestTodo(FAQUrl.APPLY_TOPIC_URL)));
return response;
}
//topic权限是否可读
if (!PermName.isReadable(topicConfig.getPerm())) {
response.setCode(ResponseCode.NO_PERMISSION);
response.setRemark("the topic[" + requestHeader.getTopic() + "] pulling message is forbidden");
return response;
}
if (requestHeader.getQueueId() < 0 || requestHeader.getQueueId() >= topicConfig.getReadQueueNums()) {
String errorInfo = String.format("queueId[%d] is illegal, topic:[%s] topicConfig.readQueueNums:[%d] consumer:[%s]",
requestHeader.getQueueId(), requestHeader.getTopic(), topicConfig.getReadQueueNums(), channel.remoteAddress());
log.warn(errorInfo);
response.setCode(ResponseCode.SYSTEM_ERROR);
response.setRemark(errorInfo);
return response;
}
SubscriptionData subscriptionData = null;
ConsumerFilterData consumerFilterData = null;
if (hasSubscriptionFlag) {
try {
subscriptionData = FilterAPI.build(
requestHeader.getTopic(), requestHeader.getSubscription(), requestHeader.getExpressionType()
);
if (!ExpressionType.isTagType(subscriptionData.getExpressionType())) {
consumerFilterData = ConsumerFilterManager.build(
requestHeader.getTopic(), requestHeader.getConsumerGroup(), requestHeader.getSubscription(),
requestHeader.getExpressionType(), requestHeader.getSubVersion()
);
assert consumerFilterData != null;
}
} catch (Exception e) {
log.warn("Parse the consumer's subscription[{}] failed, group: {}", requestHeader.getSubscription(),
requestHeader.getConsumerGroup());
response.setCode(ResponseCode.SUBSCRIPTION_PARSE_FAILED);
response.setRemark("parse the consumer's subscription failed");
return response;
}
} else {
//消费者组信息是否为空
ConsumerGroupInfo consumerGroupInfo =
this.brokerController.getConsumerManager().getConsumerGroupInfo(requestHeader.getConsumerGroup());
if (null == consumerGroupInfo) {
log.warn("the consumer's group info not exist, group: {}", requestHeader.getConsumerGroup());
response.setCode(ResponseCode.SUBSCRIPTION_NOT_EXIST);
response.setRemark("the consumer's group info not exist" + FAQUrl.suggestTodo(FAQUrl.SAME_GROUP_DIFFERENT_TOPIC));
return response;
}
if (!subscriptionGroupConfig.isConsumeBroadcastEnable()
&& consumerGroupInfo.getMessageModel() == MessageModel.BROADCASTING) {
response.setCode(ResponseCode.NO_PERMISSION);
response.setRemark("the consumer group[" + requestHeader.getConsumerGroup() + "] can not consume by broadcast way");
return response;
}
//请求主题的订阅数据是否为空
subscriptionData = consumerGroupInfo.findSubscriptionData(requestHeader.getTopic());
if (null == subscriptionData) {
log.warn("the consumer's subscription not exist, group: {}, topic:{}", requestHeader.getConsumerGroup(), requestHeader.getTopic());
response.setCode(ResponseCode.SUBSCRIPTION_NOT_EXIST);
response.setRemark("the consumer's subscription not exist" + FAQUrl.suggestTodo(FAQUrl.SAME_GROUP_DIFFERENT_TOPIC));
return response;
}
if (subscriptionData.getSubVersion() < requestHeader.getSubVersion()) {
log.warn("The broker's subscription is not latest, group: {} {}", requestHeader.getConsumerGroup(),
subscriptionData.getSubString());
response.setCode(ResponseCode.SUBSCRIPTION_NOT_LATEST);
response.setRemark("the consumer's subscription not latest");
return response;
}
if (!ExpressionType.isTagType(subscriptionData.getExpressionType())) {
consumerFilterData = this.brokerController.getConsumerFilterManager().get(requestHeader.getTopic(),
requestHeader.getConsumerGroup());
if (consumerFilterData == null) {
response.setCode(ResponseCode.FILTER_DATA_NOT_EXIST);
response.setRemark("The broker's consumer filter data is not exist!Your expression may be wrong!");
return response;
}
if (consumerFilterData.getClientVersion() < requestHeader.getSubVersion()) {
log.warn("The broker's consumer filter data is not latest, group: {}, topic: {}, serverV: {}, clientV: {}",
requestHeader.getConsumerGroup(), requestHeader.getTopic(), consumerFilterData.getClientVersion(), requestHeader.getSubVersion());
response.setCode(ResponseCode.FILTER_DATA_NOT_LATEST);
response.setRemark("the consumer's consumer filter data not latest");
return response;
}
}
}
if (!ExpressionType.isTagType(subscriptionData.getExpressionType())
&& !this.brokerController.getBrokerConfig().isEnablePropertyFilter()) {
response.setCode(ResponseCode.SYSTEM_ERROR);
response.setRemark("The broker does not support consumer to filter message by " + subscriptionData.getExpressionType());
return response;
}
// 消息过滤器 下面为创建MessageFilter逻辑，校验逻辑在这里已经结束了
MessageFilter messageFilter;
if (this.brokerController.getBrokerConfig().isFilterSupportRetry()) {
messageFilter = new ExpressionForRetryMessageFilter(subscriptionData, consumerFilterData,
this.brokerController.getConsumerFilterManager());
} else {
messageFilter = new ExpressionMessageFilter(subscriptionData, consumerFilterData,
this.brokerController.getConsumerFilterManager());
}
// 查询消息的核心入口
// 参数1：consumerGroup 消费者组
// 参数2：topic 主题
// 参数3：queueId
// 参数4：queueOffset
// 参数5：maxMsgNums 本次查询最多可拿的消息数量
// 参数6：messageFilter 上一步创建的消息过滤器（服务器这里，一般都是tagCode过滤..，getMessage方法通过consumerQueue拿到ConsumerQueueData，
// 该对象有第三个字段tagCode，和messageFilter做个匹配，如果匹配到则去commitLog里拿到这条数据，如果匹配返回false，直接跳过这条消息，不再去
// commitL里查询了）
final GetMessageResult getMessageResult =
this.brokerController.getMessageStore().getMessage(requestHeader.getConsumerGroup(), requestHeader.getTopic(),
requestHeader.getQueueId(), requestHeader.getQueueOffset(), requestHeader.getMaxMsgNums(), messageFilter);
if (getMessageResult != null) {
response.setRemark(getMessageResult.getStatus().name());
// 设置responseHeader.nextBeginOffset 为 服务器计算出来的 该队列 下一次pull时使用的offset,假如拉消息之前的offset是10，
// 拉取了5个之后的offset就是15
responseHeader.setNextBeginOffset(getMessageResult.getNextBeginOffset());
// 设置responseHeader.minOffset 为 pull queue的最小offset
responseHeader.setMinOffset(getMessageResult.getMinOffset());
// 设置responseHeader.maxOffset 为 pull queue的最大offset
responseHeader.setMaxOffset(getMessageResult.getMaxOffset());
// 设置客户端下次拉该queue时推荐使用的 brokerId
if (getMessageResult.isSuggestPullingFromSlave()) {
responseHeader.setSuggestWhichBrokerId(subscriptionGroupConfig.getWhichBrokerWhenConsumeSlowly());
} else {
responseHeader.setSuggestWhichBrokerId(MixAll.MASTER_ID);
}
// 这里代码的逻辑：如果当前主机节点角色为 slave 并且 从节点读 并未开启的话，直接给客户端 一个状态“PULL_RETRY_IMMEDIATELY”
// 客户端 检查是该状态后，会重新立马再次发起pull，此时使用的brokerId 为 MASTER_ID 了。
switch (this.brokerController.getMessageStoreConfig().getBrokerRole()) {
case ASYNC_MASTER:
case SYNC_MASTER:
break;
case SLAVE:
if (!this.brokerController.getBrokerConfig().isSlaveReadEnable()) {
response.setCode(ResponseCode.PULL_RETRY_IMMEDIATELY);
responseHeader.setSuggestWhichBrokerId(MixAll.MASTER_ID);
}
break;
}
if (this.brokerController.getBrokerConfig().isSlaveReadEnable()) {
// consume too slow ,redirect to another machine
if (getMessageResult.isSuggestPullingFromSlave()) {
responseHeader.setSuggestWhichBrokerId(subscriptionGroupConfig.getWhichBrokerWhenConsumeSlowly());
}
// consume ok
else {
responseHeader.setSuggestWhichBrokerId(subscriptionGroupConfig.getBrokerId());
}
} else {
responseHeader.setSuggestWhichBrokerId(MixAll.MASTER_ID);
}
// 根据getMessageResult的状态 设置 response的code ，，参考上一篇的状态码原理图，这块代码很混乱.
switch (getMessageResult.getStatus()) {
case FOUND:
response.setCode(ResponseCode.SUCCESS);
break;
case MESSAGE_WAS_REMOVING:
response.setCode(ResponseCode.PULL_RETRY_IMMEDIATELY);
break;
case NO_MATCHED_LOGIC_QUEUE:
case NO_MESSAGE_IN_QUEUE:
if (0 != requestHeader.getQueueOffset()) {
response.setCode(ResponseCode.PULL_OFFSET_MOVED);
// XXX: warn and notify me
log.info("the broker store no queue data, fix the request offset {} to {}, Topic: {} QueueId: {} Consumer Group: {}",
requestHeader.getQueueOffset(),
getMessageResult.getNextBeginOffset(),
requestHeader.getTopic(),
requestHeader.getQueueId(),
requestHeader.getConsumerGroup()
);
} else {
response.setCode(ResponseCode.PULL_NOT_FOUND);
}
break;
case NO_MATCHED_MESSAGE:
response.setCode(ResponseCode.PULL_RETRY_IMMEDIATELY);
break;
case OFFSET_FOUND_NULL:
response.setCode(ResponseCode.PULL_NOT_FOUND);
break;
case OFFSET_OVERFLOW_BADLY:
response.setCode(ResponseCode.PULL_OFFSET_MOVED);
// XXX: warn and notify me
log.info("the request offset: {} over flow badly, broker max offset: {}, consumer: {}",
requestHeader.getQueueOffset(), getMessageResult.getMaxOffset(), channel.remoteAddress());
break;
case OFFSET_OVERFLOW_ONE:
response.setCode(ResponseCode.PULL_NOT_FOUND);
break;
case OFFSET_TOO_SMALL:
response.setCode(ResponseCode.PULL_OFFSET_MOVED);
log.info("the request offset too small. group={}, topic={}, requestOffset={}, brokerMinOffset={}, clientIp={}",
requestHeader.getConsumerGroup(), requestHeader.getTopic(), requestHeader.getQueueOffset(),
getMessageResult.getMinOffset(), channel.remoteAddress());
break;
default:
assert false;
break;
}
// 经检查发现 服务器端并没有注册 消费hook，这里这个hook是服务器开发人员 留给自己的扩展接口
if (this.hasConsumeMessageHook()) {
ConsumeMessageContext context = new ConsumeMessageContext();
context.setConsumerGroup(requestHeader.getConsumerGroup());
context.setTopic(requestHeader.getTopic());
context.setQueueId(requestHeader.getQueueId());
String owner = request.getExtFields().get(BrokerStatsManager.COMMERCIAL_OWNER);
switch (response.getCode()) {
case ResponseCode.SUCCESS:
int commercialBaseCount = brokerController.getBrokerConfig().getCommercialBaseCount();
int incValue = getMessageResult.getMsgCount4Commercial() * commercialBaseCount;
context.setCommercialRcvStats(BrokerStatsManager.StatsType.RCV_SUCCESS);
context.setCommercialRcvTimes(incValue);
context.setCommercialRcvSize(getMessageResult.getBufferTotalSize());
context.setCommercialOwner(owner);
break;
case ResponseCode.PULL_NOT_FOUND:
if (!brokerAllowSuspend) {
context.setCommercialRcvStats(BrokerStatsManager.StatsType.RCV_EPOLLS);
context.setCommercialRcvTimes(1);
context.setCommercialOwner(owner);
}
break;
case ResponseCode.PULL_RETRY_IMMEDIATELY:
case ResponseCode.PULL_OFFSET_MOVED:
context.setCommercialRcvStats(BrokerStatsManager.StatsType.RCV_EPOLLS);
context.setCommercialRcvTimes(1);
context.setCommercialOwner(owner);
break;
default:
assert false;
break;
}
this.executeConsumeMessageHookBefore(context);
}
switch (response.getCode()) {
case ResponseCode.SUCCESS:// 查询成功
this.brokerController.getBrokerStatsManager().incGroupGetNums(requestHeader.getConsumerGroup(), requestHeader.getTopic(),
getMessageResult.getMessageCount());
this.brokerController.getBrokerStatsManager().incGroupGetSize(requestHeader.getConsumerGroup(), requestHeader.getTopic(),
getMessageResult.getBufferTotalSize());
this.brokerController.getBrokerStatsManager().incBrokerGetNums(getMessageResult.getMessageCount());
if (this.brokerController.getBrokerConfig().isTransferMsgByHeap()) {
final long beginTimeMills = this.brokerController.getMessageStore().now();
// 本次pull出来的全部消息 byte数组 表示
final byte[] r = this.readGetMessageResult(getMessageResult, requestHeader.getConsumerGroup(), requestHeader.getTopic(), requestHeader.getQueueId());
this.brokerController.getBrokerStatsManager().incGroupGetLatency(requestHeader.getConsumerGroup(),
requestHeader.getTopic(), requestHeader.getQueueId(),
(int) (this.brokerController.getMessageStore().now() - beginTimeMills));
// 将 消息 byte数组 保存到 response body字段
response.setBody(r);
} else {
try {
FileRegion fileRegion =
new ManyMessageTransfer(response.encodeHeader(getMessageResult.getBufferTotalSize()), getMessageResult);
channel.writeAndFlush(fileRegion).addListener(new ChannelFutureListener() {
@Override
public void operationComplete(ChannelFuture future) throws Exception {
getMessageResult.release();
if (!future.isSuccess()) {
log.error("transfer many message by pagecache failed, {}", channel.remoteAddress(), future.cause());
}
}
});
} catch (Throwable e) {
log.error("transfer many message by pagecache exception", e);
getMessageResult.release();
}
response = null;
}
break;
case ResponseCode.PULL_NOT_FOUND:
// 大部分情况下都是 pullRequest.offset == queue.maxOffset(客户端消费进度和服务器消息进度持平了)
// queue maxOffset 计算方式： dataSize / unit_size(20) => 得出来的结果
// 举例子，当dataSize 内存储一条 CQData 时， dataSize = 20
// dataSize / unit_size(20) => 1
// 当客户端 offset == 1 时，来到服务器拉消息，getMessage(..)方法 首先查询 CQ 数据，
// offset * unit_size => 20 这个物理位点
// 到该位点 查询数据，肯定得到 null，因为CQ的有效数据范围 [0,20]
// 所以，这种情况下 状态为 ：PULL_NOT_FOUND
// 这里需要进行长轮询，不然的话，直接返回给客户端，客户单那边逻辑 会再次立马发起 pull 请求，
// 这样会频繁拉取服务器，导致 服务器 压力很大...最好的办法就是长轮询。
// 条件成立：说明本次请求允许长轮询
// （注意brokerAllowSuspend，它是由重载的方法 传的 true，当长轮询结束时 再次 执行 processRequest的时候，该参数传的是 false）
// 也就是每次pull请求，至多在服务器端 长轮询 控制一次
if (brokerAllowSuspend && hasSuspendFlag) {
// 获取长轮询时间 15000 毫秒
long pollingTimeMills = suspendTimeoutMillisLong;
if (!this.brokerController.getBrokerConfig().isLongPollingEnable()) {
pollingTimeMills = this.brokerController.getBrokerConfig().getShortPollingTimeMills();
}
// 拉消息请求的“主题”
String topic = requestHeader.getTopic();
// 拉消息请求的“偏移量”
long offset = requestHeader.getQueueOffset();
// 拉消息请求的“队列ID”
int queueId = requestHeader.getQueueId();
// 创建 长轮询 pullRequest 对象
PullRequest pullRequest = new PullRequest(request, channel, pollingTimeMills,
this.brokerController.getMessageStore().now(), offset, subscriptionData, messageFilter);
// 将“长轮询pullRequest”对象 交给 长轮询服务
this.brokerController.getPullRequestHoldService().suspendPullRequest(topic, queueId, pullRequest);
// 将response 设置为了 null ，设置成null之后，外层 requestTask 拿到的结果就是 null，
// 它是null的话，requestTask 内部的callBack 就不会给客户端发送任何数据了..
response = null;
break;
}
case ResponseCode.PULL_RETRY_IMMEDIATELY:
break;
case ResponseCode.PULL_OFFSET_MOVED:
if (this.brokerController.getMessageStoreConfig().getBrokerRole() != BrokerRole.SLAVE
|| this.brokerController.getMessageStoreConfig().isOffsetCheckInSlave()) {
MessageQueue mq = new MessageQueue();
mq.setTopic(requestHeader.getTopic());
mq.setQueueId(requestHeader.getQueueId());
mq.setBrokerName(this.brokerController.getBrokerConfig().getBrokerName());
OffsetMovedEvent event = new OffsetMovedEvent();
event.setConsumerGroup(requestHeader.getConsumerGroup());
event.setMessageQueue(mq);
event.setOffsetRequest(requestHeader.getQueueOffset());
event.setOffsetNew(getMessageResult.getNextBeginOffset());
this.generateOffsetMovedEvent(event);
log.warn(
"PULL_OFFSET_MOVED:correction offset. topic={}, groupId={}, requestOffset={}, newOffset={}, suggestBrokerId={}",
requestHeader.getTopic(), requestHeader.getConsumerGroup(), event.getOffsetRequest(), event.getOffsetNew(),
responseHeader.getSuggestWhichBrokerId());
} else {
responseHeader.setSuggestWhichBrokerId(subscriptionGroupConfig.getBrokerId());
response.setCode(ResponseCode.PULL_RETRY_IMMEDIATELY);
log.warn("PULL_OFFSET_MOVED:none correction. topic={}, groupId={}, requestOffset={}, suggestBrokerId={}",
requestHeader.getTopic(), requestHeader.getConsumerGroup(), requestHeader.getQueueOffset(),
responseHeader.getSuggestWhichBrokerId());
}
break;
default:
assert false;
}
} else {
response.setCode(ResponseCode.SYSTEM_ERROR);
response.setRemark("store getMessage return null");
}
// 1. brokerAllowSuspend == true
// 2. sysFlag 表示提交消费者本地该queue的offset
// 3. 当前broker节点角色为 master 节点
// 三个条件全部成立时，才在broker端存储该消费者组内该queue的消费进度
boolean storeOffsetEnable = brokerAllowSuspend;
storeOffsetEnable = storeOffsetEnable && hasCommitOffsetFlag;
storeOffsetEnable = storeOffsetEnable
&& this.brokerController.getMessageStoreConfig().getBrokerRole() != BrokerRole.SLAVE;
if (storeOffsetEnable) {
// 存储该消费者组下该主题的指定队列的消费进度
this.brokerController.getConsumerOffsetManager().commitOffset(RemotingHelper.parseChannelRemoteAddr(channel),
requestHeader.getConsumerGroup(), requestHeader.getTopic(), requestHeader.getQueueId(), requestHeader.getCommitOffset());
}
// 返回response，当response不为null时，外层requestTask的callback 会将数据写给客户端
return response;
}
1.2 GetMessageResult对象主要构成
public class GetMessageResult {
// 查询消息时，最底层都是 mappedFile 支持的查询，它查询时返回给外层一个 SelectMappedBufferResult，
// mappedFile每查询一次 都会 refCount ++ ，通过SelectMappedBufferResult持有mappedFile，完成资源释放的句柄。
private final List<SelectMappedBufferResult> messageMapedList =
new ArrayList<SelectMappedBufferResult>(100);
// 该List内存储消息，每一条消息都被转成 ByteBuffer 表示了
private final List<ByteBuffer> messageBufferList = new ArrayList<ByteBuffer>(100);
// 查询结果状态 FOUND、NO_MATCHED_MESSAGE,MESSAGE_WAS_REMOVING,OFFSET_FOUND_NULL,OFFSET_OVERFLOW_BADLY......
private GetMessageStatus status;
// 客户端下次再向当前Queue拉消息时，使用的 offset
private long nextBeginOffset;
// 当前queue最小offset
private long minOffset;
// 当前queue最大offset
private long maxOffset;
// 消息总byte大小
private int bufferTotalSize = 0;
// 服务器建议客户端下次到该queue 拉消息时 使用 主/从 节点
private boolean suggestPullingFromSlave = false;
}
1.3 PullRequest对象
public class PullRequest {
// 客户端请求RemotingCommand对象
private final RemotingCommand requestCommand;
// 服务器和客户端的会话Channel
private final Channel clientChannel;
// 长轮询超时限制 15 秒
private final long timeoutMillis;
// 长轮询开始时间
private final long suspendTimestamp;
// requestCommand header 提取出来的 本次 pull 队列的 offset
private final long pullFromThisOffset;
// 该主题的订阅数据
private final SubscriptionData subscriptionData;
// 消息过滤器，一般都是 tagCode过滤
private final MessageFilter messageFilter;
public PullRequest(RemotingCommand requestCommand, Channel clientChannel, long timeoutMillis, long suspendTimestamp,
long pullFromThisOffset, SubscriptionData subscriptionData,
MessageFilter messageFilter) {
this.requestCommand = requestCommand;
this.clientChannel = clientChannel;
// 当前系统时间
this.timeoutMillis = timeoutMillis;
this.suspendTimestamp = suspendTimestamp;
this.pullFromThisOffset = pullFromThisOffset;
this.subscriptionData = subscriptionData;
this.messageFilter = messageFilter;
}
//......省略
}
2. 长轮询相关源码学习
长轮询服务的相关逻辑在PullRequestHoldService中，接下来主要看PullRequestHoldService的run方法
2.1 run方法的实现
@Override
public void run() {
log.info("{} service started", this.getServiceName());
while (!this.isStopped()) {
try {
if (this.brokerController.getBrokerConfig().isLongPollingEnable()) {
// 服务器开启长轮询开关：每次循环休眠5秒
this.waitForRunning(5 * 1000);
} else {
// 服务器关闭长轮询开关：每次循环休眠1秒
this.waitForRunning(this.brokerController.getBrokerConfig().getShortPollingTimeMills());
}
long beginLockTimestamp = this.systemClock.now();
this.checkHoldRequest();//核心逻辑
long costTime = this.systemClock.now() - beginLockTimestamp;
if (costTime > 5 * 1000) {
log.info("[NOTIFYME] check hold request cost {} ms.", costTime);
}
} catch (Throwable e) {
log.warn(this.getServiceName() + " service has exception. ", e);
}
}
log.info("{} service end", this.getServiceName());
}
run方法中，核心方法为checkHoldRequest方法没接下来看该方法的实现。
2.2 checkHoldRequest方法的实现
private void checkHoldRequest() {
for (String key : this.pullRequestTable.keySet()) {
// 循环体内为每个 topic@queueId k-v 的处理逻辑
// key 按照@拆分，得到 topic 和 queueId
String[] kArray = key.split(TOPIC_QUEUEID_SEPARATOR);
if (2 == kArray.length) {
// 主题
String topic = kArray[0];
// queueId
int queueId = Integer.parseInt(kArray[1]);
//到存储模块查询该ConsumeQueue的最大offset 如何获取？获取ConsumeQueue中MappedFile最后一个的文件名 + 文件正在顺序写的数据位点 =》 maxOffset
final long offset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId);
try {
// 通知消息达到的逻辑
// 参数1：主题
// 参数2：queueId
// 参数3：offset，当前queue最大offset
this.notifyMessageArriving(topic, queueId, offset);
} catch (Throwable e) {
log.error("check hold request failed. topic={}, queueId={}", topic, queueId, e);
}
}
}
}
2.3 notifyMessageArriving方法实现
notifyMessageArriving方法为通知消息达到的逻辑。
// 通知消息达到的逻辑
// 参数1：主题
// 参数2：queueId
// 参数3：offset，当前queue最大offset
public void notifyMessageArriving(final String topic, final int queueId, final long maxOffset) {
notifyMessageArriving(topic, queueId, maxOffset, null, 0, null, null);
}
// 该方法有两个调用点：
// 1. pullRequestHoldService.run()..
// 2. ReputMessageService 异步构建 ConsumeQueue 和 index 的消息转发服务
public void notifyMessageArriving(final String topic, final int queueId, final long maxOffset, final Long tagsCode,
long msgStoreTime, byte[] filterBitMap, Map<String, String> properties) {
// 构建key，规则：主题@queueId
String key = this.buildKey(topic, queueId);
// 获取“主题@queueId”的manyPullRequest对象
ManyPullRequest mpr = this.pullRequestTable.get(key);
if (mpr != null) {
// 获取该queue下的 pullRequest list 数据
List<PullRequest> requestList = mpr.cloneListAndClear();
if (requestList != null) {
// 重放列表，当某个 pullRequest 即不超时，
// 对应的queue的maxOffset <= pullRequest.offset 的话，就将该pullRequest 再放入replayList
List<PullRequest> replayList = new ArrayList<PullRequest>();
for (PullRequest request : requestList) {
long newestOffset = maxOffset;
if (newestOffset <= request.getPullFromThisOffset()) {
// 保证newestOffset为 queue的 maxOffset
newestOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId);
}
// 条件成立：说明该request关注的queue内有 本次pull 查询的数据了，长轮询该结束了
if (newestOffset > request.getPullFromThisOffset()) {
boolean match = request.getMessageFilter().isMatchedByConsumeQueue(tagsCode,
new ConsumeQueueExt.CqExtUnit(tagsCode, msgStoreTime, filterBitMap));
// match by bit map, need eval again when properties is not null.
if (match && properties != null) {
match = request.getMessageFilter().isMatchedByCommitLog(null, properties);
}
if (match) {
try {
// 将满足条件的 pullRequest 再次封装出 RequestTask 提交到 线程池内执行，
// 会再次调用 PullMessageProcess.processRequest(...) 三个参数的方法，并且 不允许再次长轮询。 this.brokerController.getPullMessageProcessor().executeRequestWhenWakeup(request.getClientChannel(),
request.getRequestCommand());
} catch (Throwable e) {
log.error("execute request when wakeup failed.", e);
}
continue;
}
}
// 判断 该 pullRequest 是否超时，超时，也是将它再次封装出 RequestTask 提交到 线程池内执行 三个参数的方法，并且 不允许再次长轮询。
if (System.currentTimeMillis() >= (request.getSuspendTimestamp() + request.getTimeoutMillis())) {
try {
this.brokerController.getPullMessageProcessor().executeRequestWhenWakeup(request.getClientChannel(),
request.getRequestCommand());
} catch (Throwable e) {
log.error("execute request when wakeup failed.", e);
}
continue;
}
//既没有超时，也没有满足条件，重新方法重试队列里去
replayList.add(request);
}
if (!replayList.isEmpty()) {
mpr.addPullRequest(replayList);
}
}
}
}
2.4 executeRequestWhenWakeup方法
在该方法内，不会再次触发长轮询，将数据发送给客户端就可以
public void executeRequestWhenWakeup(final Channel channel,
final RemotingCommand request) throws RemotingCommandException {
Runnable run = new Runnable() {
@Override
public void run() {
try {
// 执行 processRequest 方法，注意：第三个参数 这里是 false了，不会再次触发 长轮询了。
final RemotingCommand response = PullMessageProcessor.this.processRequest(channel, request, false);
if (response != null) {
response.setOpaque(request.getOpaque());
response.markResponseType();
try {
// 将结果数据发送给客户端
channel.writeAndFlush(response).addListener(new ChannelFutureListener() {
@Override
public void operationComplete(ChannelFuture future) throws Exception {
if (!future.isSuccess()) {
log.error("processRequestWrapper response to {} failed",
future.channel().remoteAddress(), future.cause());
log.error(request.toString());
log.error(response.toString());
}
}
});
} catch (Throwable e) {
log.error("processRequestWrapper process request over, but response failed", e);
log.error(request.toString());
log.error(response.toString());
}
}
} catch (RemotingCommandException e1) {
log.error("excuteRequestWhenWakeup run", e1);
}
}
};
this.brokerController.getPullMessageExecutor().submit(new RequestTask(run, channel, request));
}
扫一扫
337
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
