文章目录
1 部署模式
1.1 单机模式
用于本地测试，启动命令如下：
# Run a broker server with local bookies and local zookeeper
./pulsar standalone
启动入口类：PulsarStandaloneStarter
主要启动了以下几个服务：
- PulsarService：broker 服务
- LocalBookeeperEnsemble：bookie & zk 服务(2.10之后支持RocksDB存储元数据)
- WorkerService：function 相关
1.2 普通模式
单独启动一个 broker 进程，启动命令如下所示：
# Run a broker server
./pulsar broker
启动入口类：PulsarBrokerStarter
主要启动了以下几个服务：
· PulsarService：内部还会启动 BrokerService。但是 PulsarService 范围更大，包括负载管理、缓存、Schema、Admin相关的 Web 服务等，属于管理流；而 BrokerService 主要负责收发消息、创建 Netty EventLoopGroup、创建内置调度器等，属于数据流。
· BookieServer：Bookkeeper 相关
· AutoRecoveryMain：Bookkeeper autorecovery相关
· StatsProvider：Metric Exporter 相关
· WorkerService：Pulsar Function 相关，可以不启动
下面以普通模式启动源码（pulsar v2.11）进行分析
2 PulsarService
2.1 PulsarBrokerStarter.main()
public static void main(String[] args) throws Exception {
DateFormat dateFormat = new SimpleDateFormat(
FixedDateFormat.FixedFormat.ISO8601_OFFSET_DATE_TIME_HHMM.getPattern());
Thread.setDefaultUncaughtExceptionHandler((thread, exception) -> {
System.out.println(String.format("%s [%s] error Uncaught exception in thread %s: %s",
dateFormat.format(new Date()), thread.getContextClassLoader(),
thread.getName(), exception.getMessage()));
exception.printStackTrace(System.out);
});
// step1: 生成 BrokerStarter 对象，加载配置文件
BrokerStarter starter = new BrokerStarter(args);
// step2-1: 注册 Shutdown Hook
Runtime.getRuntime().addShutdownHook(
new Thread(() -> {
try {
starter.shutdown();
} catch (Throwable t) {
log.error("Error while shutting down Pulsar service", t);
} finally {
LogManager.shutdown();
}
}, "pulsar-service-shutdown")
);
// step2-2: 注册 OOM 监听器
PulsarByteBufAllocator.registerOOMListener(oomException -> {
if (starter.brokerConfig.isSkipBrokerShutdownOnOOM()) {
log.error("-- Received OOM exception: {}", oomException.getMessage(), oomException);
} else {
log.error("-- Shutting down - Received OOM exception: {}", oomException.getMessage(), oomException);
starter.pulsarService.shutdownNow();
}
});
try {
starter.start();
} catch (Throwable t) {
log.error("Failed to start pulsar service.", t);
ShutdownUtil.triggerImmediateForcefulShutdown();
} finally {
starter.join();
}
}
step1: 加载配置文件。Broker 启动时会读取 conf/broker.conf 这个配置文件，并把文件中的内容全部转化为KeyValue 的形式，然后通过反射创建一个 ServiceConfiguration 对象，并把配置文件中的值设置到 ServiceConfiguration 中的同名属性中。
step2: 注册 Shutdown Hook 和 OOM 监听器。Shutdown Hook 在监听到 Broker 进程正常退出时，会做一些清理工作。OOM 监听器则在监听到 Broker OOM 时，把异常的信息打印出来。
2.2 BrokerStarter.start()
public void start() throws Exception {
// step3: 启动 bookieStatsProvider
if (bookieStatsProvider != null) {
bookieStatsProvider.start(bookieConfig);
log.info("started bookieStatsProvider.");
}
// step4: 启动 bookieStatsProvider
if (bookieServer != null) {
bookieStartFuture = ComponentStarter.startComponent(bookieServer);
log.info("started bookieServer.");
}
if (autoRecoveryMain != null) {
autoRecoveryMain.start();
log.info("started bookie autoRecoveryMain.");
}
// 在 BrokerStarter 构造函数中创建 pulsarService 对象，然后在这调用 start() 启动
pulsarService.start();
log.info("PulsarService started.");
}
step3: 启动 BookieStatsProvider。BookieStatsProvider 对外提供了 Bookie 的Metrics 信息，这些信息和 Prometheus 的监控相关。（可选，默认不启动，通过 --run-bookie 或者 --run-bookie-autorecovery 控制）
step4: 启动 BookieServer。让 BookieServer 和 Broker 一起启动，有些特殊场景下（Standalone）可能会使用，但这样会丧失 Plusar 的存算分离特性，通常仅用于开发测试。（可选，默认不启动，通过 --run-bookie 控制）
2.3 PulsarService.start()
step5: 启动 PulsarService。内部还会启动 BrokerService。但是 PulsarService 范围更大，包括负载管理、缓存、Schema、Admin相关的 Web 服务等，属于管理流；而 BrokerService 主要负责收发消息、创建 Netty EventLoopGroup、创建内置调度器等，属于数据流。
PulsarService.start() 中创建的服务与对象包括：
详细可见《
深入解析 Apache Pulsar》P94
| 序号 | 对象 | 作用 |
|---|---|---|
| 01 | ProtocolHandlers | 支持不同protocol处理（kafka协议等） |
| 02 | PulsarResource | 用于元数据管理 |
| 03 | BrokerService | 数据流相关的服务都在这里启动 |
| … | … | … |
3 BrokerService
3.1 BrokerService.start()
public void start() throws Exception {
// producer id 分布式生成器
this.producerNameGenerator = new DistributedIdGenerator(pulsar.getCoordinationService(),
PRODUCER_NAME_GENERATOR_PATH, pulsar.getConfiguration().getClusterName());
ServiceConfiguration serviceConfig = pulsar.getConfiguration();
List<BindAddress> bindAddresses = BindAddressValidator.validateBindAddresses(serviceConfig,
Arrays.asList("pulsar", "pulsar+ssl"));
String internalListenerName = serviceConfig.getInternalListenerName();
// create a channel for each bind address
if (bindAddresses.size() == 0) {
throw new IllegalArgumentException("At least one broker bind address must be configured");
}
for (BindAddress a : bindAddresses) {
InetSocketAddress addr = new InetSocketAddress(a.getAddress().getHost(), a.getAddress().getPort());
boolean isTls = "pulsar+ssl".equals(a.getAddress().getScheme());
PulsarChannelInitializer.PulsarChannelOptions opts = PulsarChannelInitializer.PulsarChannelOptions.builder()
.enableTLS(isTls)
.listenerName(a.getListenerName()).build();
// step1: 启动器，负责装配 netty 组件，启动并提供服务
ServerBootstrap b = defaultServerBootstrap.clone();
// step2: childHandler 负责处理读写，该方法决定了 child 执行哪些操作
// ChannelInitializer 处理器（仅执行一次），它的作用是待客户端 SocketChannel 建立连接后，执行initChannel 以便添加更多的处理器
b.childHandler(
pulsarChannelInitFactory.newPulsarChannelInitializer(pulsar, opts));
try {
// 绑定端口
Channel ch = b.bind(addr).sync().channel();
listenChannels.add(ch);
// identify the primary channel. Note that the legacy bindings appear first and have no listener.
if (StringUtils.isBlank(a.getListenerName())
|| StringUtils.equalsIgnoreCase(a.getListenerName(), internalListenerName)) {
if (this.listenChannel == null && !isTls) {
this.listenChannel = ch;
}
if (this.listenChannelTls == null && isTls) {
this.listenChannelTls = ch;
}
}
log.info("Started Pulsar Broker service on {}, TLS: {}, listener: {}",
ch.localAddress(),
isTls ? SslContext.defaultServerProvider().toString() : "(none)",
StringUtils.defaultString(a.getListenerName(), "(none)"));
} catch (Exception e) {
throw new IOException("Failed to bind Pulsar broker on " + addr, e);
}
}
// start other housekeeping functions
this.startStatsUpdater(
serviceConfig.getStatsUpdateInitialDelayInSecs(),
serviceConfig.getStatsUpdateFrequencyInSecs());
// step3: 启动一堆需要定期执行的任务，包括：长时间无效的topic、长时间无效的producer（和message去重相关）、长时间无效的subscription等
this.startInactivityMonitor();
this.startMessageExpiryMonitor();
this.startCompactionMonitor();
this.startConsumedLedgersMonitor();
this.startBacklogQuotaChecker();
this.updateBrokerPublisherThrottlingMaxRate();
this.updateBrokerDispatchThrottlingMaxRate();
this.startCheckReplicationPolicies();
this.startDeduplicationSnapshotMonitor();
}
step1: 创建 netty 启动器。
step2: netty 启动器上设置 childHandler(ChannelInitializer(上面包含一系列处理器))。
step3: 启动一堆需要定期执行的任务，包括：长时间无效的topic、长时间无效的producer（和message去重相关）、长时间无效的subscription等。
3.2 PulsarChannelInitializer.initChannel()
顺着 netty 的初始化方式我们直接看 ChannelInitializer - PulsarChannelInitializer，这里应该和 Kafka 类似进行处理请求的操作。
重写 initChannel() 方法，在 SocketChannel 上添加处理器，这里最重要的业务处理器是 ServerCnx。
@Override
protected void initChannel(SocketChannel ch) throws Exception {
ch.pipeline().addLast("consolidation", new FlushConsolidationHandler(1024, true));
if (this.enableTls) {
if (this.tlsEnabledWithKeyStore) {
ch.pipeline().addLast(TLS_HANDLER,
new SslHandler(nettySSLContextAutoRefreshBuilder.get().createSSLEngine()));
} else {
ch.pipeline().addLast(TLS_HANDLER, sslCtxRefresher.get().newHandler(ch.alloc()));
}
ch.pipeline().addLast("ByteBufPairEncoder", ByteBufPair.COPYING_ENCODER);
} else {
ch.pipeline().addLast("ByteBufPairEncoder", ByteBufPair.ENCODER);
}
if (pulsar.getConfiguration().isHaProxyProtocolEnabled()) {
ch.pipeline().addLast(OptionalProxyProtocolDecoder.NAME, new OptionalProxyProtocolDecoder());
}
ch.pipeline().addLast("frameDecoder", new LengthFieldBasedFrameDecoder(
brokerConf.getMaxMessageSize() + Commands.MESSAGE_SIZE_FRAME_PADDING, 0, 4, 0, 4));
// https://stackoverflow.com/questions/37535482/netty-disabling-auto-read-doesnt-work-for-bytetomessagedecoder
// Classes such as {@link ByteToMessageDecoder} or {@link MessageToByteEncoder} are free to emit as many events
// as they like for any given input. so, disabling auto-read on `ByteToMessageDecoder` doesn't work properly and
// ServerCnx ends up reading higher number of messages and broker can not throttle the messages by disabling
// auto-read.
ch.pipeline().addLast("flowController", new FlowControlHandler());
// ***** SocketChannel的业务处理器
ServerCnx cnx = newServerCnx(pulsar, listenerName);
ch.pipeline().addLast("handler", cnx);
connections.put(ch.remoteAddress(), cnx);
}
3.3 ServerCnx.handleXxx()
这个类的作用可以对标 KafkaApis，处理各种 Api 请求，实际上是一个ChannelHandler，继承了 PulsarHandler（主要负责一些连接的 keepalive 逻辑）。
PulsarHandler 继承了 PulsarDecoder （ 主要负责序列化，反序列化 Api 请求），PulsarDecoder实际上是一个 ChannelInboundHandlerAdapter。
@Override
public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
if (msg instanceof HAProxyMessage) {
HAProxyMessage proxyMessage = (HAProxyMessage) msg;
this.proxyMessage = proxyMessage;
proxyMessage.release();
return;
}
// Get a buffer that contains the full frame
ByteBuf buffer = (ByteBuf) msg;
try {
// De-serialize the command
int cmdSize = (int) buffer.readUnsignedInt();
cmd.parseFrom(buffer, cmdSize);
if (log.isDebugEnabled()) {
log.debug("[{}] Received cmd {}", ctx.channel().remoteAddress(), cmd.getType());
}
messageReceived();
switch (cmd.getType()) {
case PARTITIONED_METADATA:
checkArgument(cmd.hasPartitionMetadata());
try {
interceptCommand(cmd);
handlePartitionMetadataRequest(cmd.getPartitionMetadata());
} catch (InterceptException e) {
ctx.writeAndFlush(Commands.newPartitionMetadataResponse(getServerError(e.getErrorCode()),
e.getMessage(), cmd.getPartitionMetadata().getRequestId()));
}
break;
case PARTITIONED_METADATA_RESPONSE:
checkArgument(cmd.hasPartitionMetadataResponse());
handlePartitionResponse(cmd.getPartitionMetadataResponse());
break;
case LOOKUP:
checkArgument(cmd.hasLookupTopic());
handleLookup(cmd.getLookupTopic());
break;
case LOOKUP_RESPONSE:
checkArgument(cmd.hasLookupTopicResponse());
handleLookupResponse(cmd.getLookupTopicResponse());
break;
case ACK:
checkArgument(cmd.hasAck());
safeInterceptCommand(cmd);
handleAck(cmd.getAck());
break;
case ACK_RESPONSE:
checkArgument(cmd.hasAckResponse());
handleAckResponse(cmd.getAckResponse());
break;
case CLOSE_CONSUMER:
checkArgument(cmd.hasCloseConsumer());
safeInterceptCommand(cmd);
handleCloseConsumer(cmd.getCloseConsumer());
break;
case CLOSE_PRODUCER:
checkArgument(cmd.hasCloseProducer());
safeInterceptCommand(cmd);
handleCloseProducer(cmd.getCloseProducer());
break;
case CONNECT:
checkArgument(cmd.hasConnect());
handleConnect(cmd.getConnect());
break;
case CONNECTED:
checkArgument(cmd.hasConnected());
handleConnected(cmd.getConnected());
break;
case ERROR:
checkArgument(cmd.hasError());
handleError(cmd.getError());
break;
case FLOW:
checkArgument(cmd.hasFlow());
handleFlow(cmd.getFlow());
break;
case MESSAGE: {
checkArgument(cmd.hasMessage());
handleMessage(cmd.getMessage(), buffer);
break;
}
case PRODUCER:
checkArgument(cmd.hasProducer());
try {
interceptCommand(cmd);
handleProducer(cmd.getProducer());
} catch (InterceptException e) {
ctx.writeAndFlush(Commands.newError(cmd.getProducer().getRequestId(),
getServerError(e.getErrorCode()), e.getMessage()));
}
break;
case SEND: {
checkArgument(cmd.hasSend());
try {
interceptCommand(cmd);
// Store a buffer marking the content + headers
ByteBuf headersAndPayload = buffer.markReaderIndex();
handleSend(cmd.getSend(), headersAndPayload);
} catch (InterceptException e) {
ctx.writeAndFlush(Commands.newSendError(cmd.getSend().getProducerId(),
cmd.getSend().getSequenceId(), getServerError(e.getErrorCode()), e.getMessage()));
}
break;
}
case SEND_ERROR:
checkArgument(cmd.hasSendError());
handleSendError(cmd.getSendError());
break;
case SEND_RECEIPT:
checkArgument(cmd.hasSendReceipt());
handleSendReceipt(cmd.getSendReceipt());
break;
case SUBSCRIBE:
checkArgument(cmd.hasSubscribe());
try {
interceptCommand(cmd);
handleSubscribe(cmd.getSubscribe());
} catch (InterceptException e) {
ctx.writeAndFlush(Commands.newError(cmd.getSubscribe().getRequestId(),
getServerError(e.getErrorCode()), e.getMessage()));
}
break;
case SUCCESS:
checkArgument(cmd.hasSuccess());
handleSuccess(cmd.getSuccess());
break;
case PRODUCER_SUCCESS:
checkArgument(cmd.hasProducerSuccess());
handleProducerSuccess(cmd.getProducerSuccess());
break;
case UNSUBSCRIBE:
checkArgument(cmd.hasUnsubscribe());
safeInterceptCommand(cmd);
handleUnsubscribe(cmd.getUnsubscribe());
break;
case SEEK:
checkArgument(cmd.hasSeek());
try {
interceptCommand(cmd);
handleSeek(cmd.getSeek());
} catch (InterceptException e) {
ctx.writeAndFlush(Commands.newError(cmd.getSeek().getRequestId(), getServerError(e.getErrorCode()),
e.getMessage()));
}
break;
case PING:
checkArgument(cmd.hasPing());
handlePing(cmd.getPing());
break;
case PONG:
checkArgument(cmd.hasPong());
handlePong(cmd.getPong());
break;
case REDELIVER_UNACKNOWLEDGED_MESSAGES:
checkArgument(cmd.hasRedeliverUnacknowledgedMessages());
handleRedeliverUnacknowledged(cmd.getRedeliverUnacknowledgedMessages());
break;
case CONSUMER_STATS:
checkArgument(cmd.hasConsumerStats());
handleConsumerStats(cmd.getConsumerStats());
break;
case CONSUMER_STATS_RESPONSE:
checkArgument(cmd.hasConsumerStatsResponse());
handleConsumerStatsResponse(cmd.getConsumerStatsResponse());
break;
case REACHED_END_OF_TOPIC:
checkArgument(cmd.hasReachedEndOfTopic());
handleReachedEndOfTopic(cmd.getReachedEndOfTopic());
break;
case TOPIC_MIGRATED:
checkArgument(cmd.hasTopicMigrated());
handleTopicMigrated(cmd.getTopicMigrated());
break;
case GET_LAST_MESSAGE_ID:
checkArgument(cmd.hasGetLastMessageId());
handleGetLastMessageId(cmd.getGetLastMessageId());
break;
case GET_LAST_MESSAGE_ID_RESPONSE:
checkArgument(cmd.hasGetLastMessageIdResponse());
handleGetLastMessageIdSuccess(cmd.getGetLastMessageIdResponse());
break;
case ACTIVE_CONSUMER_CHANGE:
handleActiveConsumerChange(cmd.getActiveConsumerChange());
break;
case GET_TOPICS_OF_NAMESPACE:
checkArgument(cmd.hasGetTopicsOfNamespace());
try {
interceptCommand(cmd);
handleGetTopicsOfNamespace(cmd.getGetTopicsOfNamespace());
} catch (InterceptException e) {
ctx.writeAndFlush(Commands.newError(cmd.getGetTopicsOfNamespace().getRequestId(),
getServerError(e.getErrorCode()), e.getMessage()));
}
break;
case GET_TOPICS_OF_NAMESPACE_RESPONSE:
checkArgument(cmd.hasGetTopicsOfNamespaceResponse());
handleGetTopicsOfNamespaceSuccess(cmd.getGetTopicsOfNamespaceResponse());
break;
case GET_SCHEMA:
checkArgument(cmd.hasGetSchema());
try {
interceptCommand(cmd);
handleGetSchema(cmd.getGetSchema());
} catch (InterceptException e) {
ctx.writeAndFlush(Commands.newGetSchemaResponseError(cmd.getGetSchema().getRequestId(),
getServerError(e.getErrorCode()), e.getMessage()));
}
break;
case GET_SCHEMA_RESPONSE:
checkArgument(cmd.hasGetSchemaResponse());
handleGetSchemaResponse(cmd.getGetSchemaResponse());
break;
case GET_OR_CREATE_SCHEMA:
checkArgument(cmd.hasGetOrCreateSchema());
try {
interceptCommand(cmd);
handleGetOrCreateSchema(cmd.getGetOrCreateSchema());
} catch (InterceptException e) {
ctx.writeAndFlush(Commands.newGetOrCreateSchemaResponseError(
cmd.getGetOrCreateSchema().getRequestId(), getServerError(e.getErrorCode()),
e.getMessage()));
}
break;
case GET_OR_CREATE_SCHEMA_RESPONSE:
checkArgument(cmd.hasGetOrCreateSchemaResponse());
handleGetOrCreateSchemaResponse(cmd.getGetOrCreateSchemaResponse());
break;
case AUTH_CHALLENGE:
checkArgument(cmd.hasAuthChallenge());
handleAuthChallenge(cmd.getAuthChallenge());
break;
case AUTH_RESPONSE:
checkArgument(cmd.hasAuthResponse());
handleAuthResponse(cmd.getAuthResponse());
break;
case TC_CLIENT_CONNECT_REQUEST:
checkArgument(cmd.hasTcClientConnectRequest());
handleTcClientConnectRequest(cmd.getTcClientConnectRequest());
break;
case TC_CLIENT_CONNECT_RESPONSE:
checkArgument(cmd.hasTcClientConnectResponse());
handleTcClientConnectResponse(cmd.getTcClientConnectResponse());
break;
case NEW_TXN:
checkArgument(cmd.hasNewTxn());
handleNewTxn(cmd.getNewTxn());
break;
case NEW_TXN_RESPONSE:
checkArgument(cmd.hasNewTxnResponse());
handleNewTxnResponse(cmd.getNewTxnResponse());
break;
case ADD_PARTITION_TO_TXN:
checkArgument(cmd.hasAddPartitionToTxn());
handleAddPartitionToTxn(cmd.getAddPartitionToTxn());
break;
case ADD_PARTITION_TO_TXN_RESPONSE:
checkArgument(cmd.hasAddPartitionToTxnResponse());
handleAddPartitionToTxnResponse(cmd.getAddPartitionToTxnResponse());
break;
case ADD_SUBSCRIPTION_TO_TXN:
checkArgument(cmd.hasAddSubscriptionToTxn());
handleAddSubscriptionToTxn(cmd.getAddSubscriptionToTxn());
break;
case ADD_SUBSCRIPTION_TO_TXN_RESPONSE:
checkArgument(cmd.hasAddSubscriptionToTxnResponse());
handleAddSubscriptionToTxnResponse(cmd.getAddSubscriptionToTxnResponse());
break;
case END_TXN:
checkArgument(cmd.hasEndTxn());
handleEndTxn(cmd.getEndTxn());
break;
case END_TXN_RESPONSE:
checkArgument(cmd.hasEndTxnResponse());
handleEndTxnResponse(cmd.getEndTxnResponse());
break;
case END_TXN_ON_PARTITION:
checkArgument(cmd.hasEndTxnOnPartition());
handleEndTxnOnPartition(cmd.getEndTxnOnPartition());
break;
case END_TXN_ON_PARTITION_RESPONSE:
checkArgument(cmd.hasEndTxnOnPartitionResponse());
handleEndTxnOnPartitionResponse(cmd.getEndTxnOnPartitionResponse());
break;
case END_TXN_ON_SUBSCRIPTION:
checkArgument(cmd.hasEndTxnOnSubscription());
handleEndTxnOnSubscription(cmd.getEndTxnOnSubscription());
break;
case END_TXN_ON_SUBSCRIPTION_RESPONSE:
checkArgument(cmd.hasEndTxnOnSubscriptionResponse());
handleEndTxnOnSubscriptionResponse(cmd.getEndTxnOnSubscriptionResponse());
break;
case WATCH_TOPIC_LIST:
checkArgument(cmd.hasWatchTopicList());
handleCommandWatchTopicList(cmd.getWatchTopicList());
break;
case WATCH_TOPIC_LIST_SUCCESS:
checkArgument(cmd.hasWatchTopicListSuccess());
handleCommandWatchTopicListSuccess(cmd.getWatchTopicListSuccess());
break;
case WATCH_TOPIC_UPDATE:
checkArgument(cmd.hasWatchTopicUpdate());
handleCommandWatchTopicUpdate(cmd.getWatchTopicUpdate());
break;
case WATCH_TOPIC_LIST_CLOSE:
checkArgument(cmd.hasWatchTopicListClose());
handleCommandWatchTopicListClose(cmd.getWatchTopicListClose());
break;
default:
break;
}
} finally {
buffer.release();
}
}
而 Pulsar APi 实际上是通过 Pulsar.proto 生成的，这里编写了各种 Api 的定义。
639
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
