概述
Kafka源码包含多个模块，每个模块负责不同的功能。以下是一些核心模块及其功能的概述：
-
服务端源码 ：实现Kafka Broker的核心功能，包括日志存储、控制器、协调器、元数据管理及状态机管理、延迟机制、消费者组管理、高并发网络架构模型实现等。
-
Java客户端源码 ：实现了Producer和Consumer与Broker的交互机制，以及通用组件支撑代码。
-
Connect源码 ：用来构建异构数据双向流式同步服务。
-
Stream源码 ：用来实现实时流处理相关功能。
-
Raft源码 ：实现了Raft一致性协议。
-
Admin模块 ：Kafka的管理员模块，操作和管理其topic，partition相关，包含创建，删除topic，或者拓展分区等。
-
Api模块 ：负责数据交互，客户端与服务端交互数据的编码与解码。
-
Client模块 ：包含Producer读取Kafka Broker元数据信息的类，如topic和分区，以及leader。
-
Cluster模块 ：包含Broker、Cluster、Partition、Replica等实体类。
-
Common模块 ：包含各种异常类以及错误验证。
-
Consumer模块 ：消费者处理模块，负责客户端消费者数据和逻辑处理。
-
Controller模块 ：负责中央控制器的选举，分区的Leader选举，Replica的分配或重新分配，分区和副本的扩容等。
-
Coordinator模块 ：负责管理部分consumer group和他们的offset。
-
Javaapi模块 ：提供Java语言的Producer和Consumer的API接口。
-
Log模块 ：负责Kafka文件存储，读写所有Topic消息数据。
-
Message模块 ：封装多条数据组成数据集或压缩数据集。
-
Metrics模块 ：负责内部状态监控。
-
Network模块 ：处理客户端连接，网络事件模块。
-
Producer模块 ：生产者细节实现，包括同步和异步消息发送。
-
Security模块 ：负责Kafka的安全验证和管理。
-
Serializer模块 ：序列化和反序列化消息内容。
-
Server模块 ：涉及Leader和Offset的checkpoint，动态配置，延时创建和删除Topic，Leader选举，Admin和Replica管理等。
-
Tools模块 ：包含多种工具，如导出consumer offset值，LogSegments信息，Topic的log位置信息，Zookeeper上的offset值等。
-
Utils模块 ：包含各种工具类，如Json，ZkUtils，线程池工具类，KafkaScheduler公共调度器类等。
这些模块共同构成了Kafka的整体架构，使其能够提供高吞吐量、高可用性的消息队列服务。
kafka源码分支为1.0.2
入口类：kafka.Kafka
object Kafka extends Logging {
def getPropsFromArgs(args: Array[String]): Properties = {
val optionParser = new OptionParser(false)
val overrideOpt = optionParser.accepts("override", "Optional property that should override values set in server.properties file")
.withRequiredArg()
.ofType(classOf[String])
if (args.length == 0) {
CommandLineUtils.printUsageAndDie(optionParser, "USAGE: java [options] %s server.properties [--override property=value]*".format(classOf[KafkaServer].getSimpleName()))
}
//第一个参数即为server.properties配置文件路径，将其中的配置项加载到props对象
val props = Utils.loadProps(args(0))
if (args.length > 1) {
val options = optionParser.parse(args.slice(1, args.length): _*)
if (options.nonOptionArguments().size() > 0) {
CommandLineUtils.printUsageAndDie(optionParser, "Found non argument parameters: " + options.nonOptionArguments().toArray.mkString(","))
}
props ++= CommandLineUtils.parseKeyValueArgs(options.valuesOf(overrideOpt).asScala)
}
props
}
private def registerLoggingSignalHandler(): Unit = {
val jvmSignalHandlers = new ConcurrentHashMap[String, SignalHandler]().asScala
val handler = new SignalHandler() {
override def handle(signal: Signal) {
info(s"Terminating process due to signal $signal")
jvmSignalHandlers.get(signal.getName).foreach(_.handle(signal))
}
}
def registerHandler(signalName: String) {
val oldHandler = Signal.handle(new Signal(signalName), handler)
if (oldHandler != null)
jvmSignalHandlers.put(signalName, oldHandler)
}
if (!OperatingSystem.IS_WINDOWS) {
registerHandler("TERM")
registerHandler("INT")
registerHandler("HUP")
}
}
def main(args: Array[String]): Unit = {
try {
//读取启动参数，并加载配置文件
val serverProps = getPropsFromArgs(args)
//将变量解析并赋值到KafkaConfig的各个字段中
val kafkaServerStartable = KafkaServerStartable.fromProps(serverProps)
// register signal handler to log termination due to SIGTERM, SIGHUP and SIGINT (control-c)
registerLoggingSignalHandler()
//注册进程关闭的钩子方法，会对各个组件做一些关闭、清理操作
// attach shutdown handler to catch terminating signals as well as normal termination
Runtime.getRuntime().addShutdownHook(new Thread("kafka-shutdown-hook") {
override def run(): Unit = kafkaServerStartable.shutdown()
})
//服务启动
kafkaServerStartable.startup()
//等待shutdown()方法执行完成
kafkaServerStartable.awaitShutdown()
}
catch {
case e: Throwable =>
fatal(e)
Exit.exit(1)
}
Exit.exit(0)
}
}
上面main()中会调用KafkaServer的startup()方法：
/**
* Start up API for bringing up a single instance of the Kafka server.
* Instantiates the LogManager, the SocketServer and the request handlers - KafkaRequestHandlers
*/
def startup() {
try {
info("starting")
//若已经调用了shutdown()方法，则不能启动
if(isShuttingDown.get) {
throw new IllegalStateException("Kafka server is still shutting down, cannot re-start!")
}
//已经启动完成，不用重复启动
if(startupComplete.get)
return
//CAS设置isStartingUp状态，防止多线程同时走到这里
val canStartup = isStartingUp.compareAndSet(false, true)
if (canStartup) {
//设置为Starting状态
brokerState.newState(Starting)
//启动kafka定时器，其他类中使用其设置定时任务
/* start scheduler */
kafkaScheduler.startup()
//初始化zk连接
/* setup zookeeper */
zkUtils = initZk()
//从zk中获取，或者生成clusterId
/* Get or create cluster_id */
_clusterId = getOrGenerateClusterId(zkUtils)
info(s"Cluster ID = $clusterId")
//生成brokerId
/* generate brokerId */
val (brokerId, initialOfflineDirs) = getBrokerIdAndOfflineDirs
config.brokerId = brokerId
//创建日志上下文
logContext = new LogContext(s"[KafkaServer id=${config.brokerId}] ")
this.logIdent = logContext.logPrefix
/* create and configure metrics */
val reporters = config.getConfiguredInstances(KafkaConfig.MetricReporterClassesProp, classOf[MetricsReporter],
Map[String, AnyRef](KafkaConfig.BrokerIdProp -> (config.brokerId.toString)).asJava)
//默认会增加jmx监控上报
reporters.add(new JmxReporter(jmxPrefix))
val metricConfig = KafkaServer.metricConfig(config)
metrics = new Metrics(metricConfig, reporters, time, true)
/* register broker metrics */
_brokerTopicStats = new BrokerTopicStats
//配置限制管理器，限制producer、consumer的拉取、消费速率等
quotaManagers = QuotaFactory.instantiate(config, metrics, time, threadNamePrefix.getOrElse(""))
notifyClusterListeners(kafkaMetricsReporters ++ reporters.asScala)
logDirFailureChannel = new LogDirFailureChannel(config.logDirs.size)
//日志管理器
/* start log manager */
logManager = LogManager(config, initialOfflineDirs, zkUtils, brokerState, kafkaScheduler, time, brokerTopicStats, logDirFailureChannel)
logManager.startup()
//元数据缓存
metadataCache = new MetadataCache(config.brokerId)
credentialProvider = new CredentialProvider(config.saslEnabledMechanisms)
//启动socker服务，用于接受客户端请求
// Create and start the socket server acceptor threads so that the bound port is known.
// Delay starting processors until the end of the initialization sequence to ensure
// that credentials have been loaded before processing authentications.
socketServer = new SocketServer(config, metrics, time, credentialProvider)
socketServer.startup(startupProcessors = false)
//副本管理器
/* start replica manager */
replicaManager = createReplicaManager(isShuttingDown)
replicaManager.startup()
//控制器，集群只有一个broker会竞选为控制器
/* start kafka controller */
kafkaController = new KafkaController(config, zkUtils, time, metrics, threadNamePrefix)
kafkaController.startup()
adminManager = new AdminManager(config, metrics, metadataCache, zkUtils)
//消费组协调器
/* start group coordinator */
// Hardcode Time.SYSTEM for now as some Streams tests fail otherwise, it would be good to fix the underlying issue
groupCoordinator = GroupCoordinator(config, zkUtils, replicaManager, Time.SYSTEM)
groupCoordinator.startup()
//事务协调器
/* start transaction coordinator, with a separate background thread scheduler for transaction expiration and log loading */
// Hardcode Time.SYSTEM for now as some Streams tests fail otherwise, it would be good to fix the underlying issue
transactionCoordinator = TransactionCoordinator(config, replicaManager, new KafkaScheduler(threads = 1, threadNamePrefix = "transaction-log-manager-"), zkUtils, metrics, metadataCache, Time.SYSTEM)
transactionCoordinator.startup()
/* Get the authorizer and initialize it if one is specified.*/
authorizer = Option(config.authorizerClassName).filter(_.nonEmpty).map { authorizerClassName =>
val authZ = CoreUtils.createObject[Authorizer](authorizerClassName)
authZ.configure(config.originals())
authZ
}
//处理各类api请求
/* start processing requests */
apis = new KafkaApis(socketServer.requestChannel, replicaManager, adminManager, groupCoordinator, transactionCoordinator,
kafkaController, zkUtils, config.brokerId, config, metadataCache, metrics, authorizer, quotaManagers,
brokerTopicStats, clusterId, time)
//请求处理池
requestHandlerPool = new KafkaRequestHandlerPool(config.brokerId, socketServer.requestChannel, apis, time,
config.numIoThreads)
Mx4jLoader.maybeLoad()
//动态配置管理器
/* start dynamic config manager */
dynamicConfigHandlers = Map[String, ConfigHandler](ConfigType.Topic -> new TopicConfigHandler(logManager, config, quotaManagers),
ConfigType.Client -> new ClientIdConfigHandler(quotaManagers),
ConfigType.User -> new UserConfigHandler(quotaManagers, credentialProvider),
ConfigType.Broker -> new BrokerConfigHandler(config, quotaManagers))
// Create the config manager. start listening to notifications
dynamicConfigManager = new DynamicConfigManager(zkUtils, dynamicConfigHandlers)
dynamicConfigManager.startup()
//所有监听器
/* tell everyone we are alive */
val listeners = config.advertisedListeners.map { endpoint =>
if (endpoint.port == 0)
endpoint.copy(port = socketServer.boundPort(endpoint.listenerName))
else
endpoint
}
//启动健康检查器，会将brokerid注册到zk
kafkaHealthcheck = new KafkaHealthcheck(config.brokerId, listeners, zkUtils, config.rack,
config.interBrokerProtocolVersion)
kafkaHealthcheck.startup()
//创建brokerid检查点，刷新到磁盘
// Now that the broker id is successfully registered via KafkaHealthcheck, checkpoint it
checkpointBrokerId(config.brokerId)
//对所有acceptor连接创建处理器
socketServer.startProcessors()
//改变broker状态为RunningAsBroker
brokerState.newState(RunningAsBroker)
//设置一些状态
shutdownLatch = new CountDownLatch(1)
startupComplete.set(true)
isStartingUp.set(false)
AppInfoParser.registerAppInfo(jmxPrefix, config.brokerId.toString, metrics)
info("started")
}
}
catch {
case e: Throwable =>
fatal("Fatal error during KafkaServer startup. Prepare to shutdown", e)
isStartingUp.set(false)
shutdown()
throw e
}
}
至此Broker进程启动完毕
1378
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
