- 本次目标 -
从这次课开始我们会有连续好几篇文章都会去剖析Kafka Server端的网络部分。Kafka Server端的网路部分设计是整个Kafka的精华部分,技术含量很高,大家注意掌握。Kafka服务启动以后首先会启动Acceptor线程,这个线程是专门用来接收客户端连接的,我们分析Kafka的服务端就从这个线程开始。- 源码剖析 -
首先看整个Kafka服务端的核心入口类Kafka,我们看一下这个类的main方法:def main(args: Array[String]): Unit = {
try {
val serverProps = getPropsFromArgs(args)
val kafkaServerStartable = KafkaServerStartable.fromProps(serverProps)
// attach shutdown handler to catch control-c
Runtime.getRuntime().addShutdownHook(new Thread() {
override def run() = {
kafkaServerStartable.shutdown
}
})
//TODO 核心代码,启动Kafka线程
kafkaServerStartable.startup
//等待结束
kafkaServerStartable.awaitShutdown
}
catch {
case e: Throwable =>
fatal(e)
System.exit(1)
}
System.exit(0)
接着往下看:
//TODO 初始化kafkaServer对象
private val server = new KafkaServer(serverConfig)
def startup() {
try {
//TODO 启动kafkaServer
server.startup()
}
catch {
case e: Throwable =>
fatal("Fatal error during KafkaServerStartable startup. Prepare to shutdown", e)
// KafkaServer already calls shutdown() internally, so this is purely for logging & the exit code
System.exit(1)
}
}
调用startup方法
def startup() {
try {
........
/* setup zookeeper */
zkUtils = initZk()
/* start log manager */
logManager = createLogManager(zkUtils.zkClient, brokerState)
logManager.startup()
/* generate brokerId */
config.brokerId = getBrokerId
this.logIdent = "[Kafka Server " + config.brokerId + "], "
/**
* 其实如果大家观察这个整个方法,见名知意
* 就感觉到这个方法里面有很多重要的方法,但是大家要知道
* 我们此次分析源码的方式使用的是场景驱动的方式,我们现在
* 主要关心的是客户端发送过来请求以后服务端这儿是如何处理的?
* 所以我们此次重要的方法是这个。
*/
socketServer = new SocketServer(config, metrics, kafkaMetricsTime)
socketServer.startup()
/* start replica manager */
replicaManager = new ReplicaManager(config, metrics, time, kafkaMetricsTime, zkUtils, kafkaScheduler, logManager,
isShuttingDown)
replicaManager.startup()
/* start kafka controller */
kafkaController = new KafkaController(config, zkUtils, brokerState, kafkaMetricsTime, metrics, threadNamePrefix)
kafkaController.startup()
/* start group coordinator */
groupCoordinator = GroupCoordinator(config, zkUtils, replicaManager, kafkaMetricsTime)
groupCoordinator.startup()
/* Get the authorizer and initialize it if one is specified.*/
authorizer = Option(config.authorizerClassName).filter(_.nonEmpty).map { authorizerClassName =>
val authZ = CoreUtils.createObject[Authorizer](authorizerClassName)
authZ.configure(config.originals())
authZ
}
/* start processing requests */
apis = new KafkaApis(socketServer.requestChannel, replicaManager, groupCoordinator,
kafkaController, zkUtils, config.brokerId, config, metadataCache, metrics, authorizer)
requestHandlerPool = new KafkaRequestHandlerPool(config.brokerId, socketServer.requestChannel, apis, config.numIoThreads)
brokerState.newState(RunningAsBroker)
Mx4jLoader.maybeLoad()
/* start dynamic config manager */
dynamicConfigHandlers = Map[String, ConfigHandler](ConfigType.Topic -> new TopicConfigHandler(logManager, config),
ConfigType.Client -> new ClientIdConfigHandler(apis.quotaManagers))
.......
}
}
调用startup方法
def startup() {
this.synchronized {
connectionQuotas = new ConnectionQuotas(maxConnectionsPerIp, maxConnectionsPerIpOverrides)
val sendBufferSize = config.socketSendBufferBytes
val recvBufferSize = config.socketReceiveBufferBytes
val brokerId = config.brokerId
var processorBeginIndex = 0
endpoints.values.foreach { endpoint =>
val protocol = endpoint.protocolType
val processorEndIndex = processorBeginIndex + numProcessorThreads
for (i
processors(i) = newProcessor(i, connectionQuotas, protocol)
//TODO 初始化Acceptor
val acceptor = new Acceptor(endpoint, sendBufferSize, recvBufferSize, brokerId,
processors.slice(processorBeginIndex, processorEndIndex), connectionQuotas)
acceptors.put(endpoint, acceptor)
//封装acceptor线程,然后启动Acceptor线程。
Utils.newThread("kafka-socket-acceptor-%s-%d".format(protocol.toString, endpoint.port), acceptor, false).start()
acceptor.awaitStartup()
processorBeginIndex = processorEndIndex
}
}
既然调用了Acceptor线程的start方法,也就是说我们接下来要看Acceptor的run方法:
def run() {
//TODO 网络Selector上注册OP_ACCEPT事件
serverChannel.register(nioSelector, SelectionKey.OP_ACCEPT)
//TODO 调用countdown
startupComplete()
try {
var currentProcessor = 0
while (isRunning) {
try {
val ready = nioSelector.select(500)
if (ready > 0) {
val keys = nioSelector.selectedKeys()
val iter = keys.iterator()
//遍历所有的key
while (iter.hasNext && isRunning) {
try {
val key = iter.next
iter.remove()
//如果有连接发送过来
if (key.isAcceptable)
//执行此方法,统一对连接进行处理,
//但是具体是怎么处理的我们下一讲继续分析。
accept(key, processors(currentProcessor))
else
throw new IllegalStateException("Unrecognized key state for acceptor thread.")
// round robin to the next processor thread
currentProcessor = (currentProcessor + 1) % processors.length
} catch {
case e: Throwable => error("Error while accepting connection", e)
}
}
}
}
catch {
// We catch all the throwables to prevent the acceptor thread from exiting on exceptions due
// to a select operation on a specific channel or a bad request. We don't want the
// the broker to stop responding to requests from other clients in these scenarios.
case e: ControlThrowable => throw e
case e: Throwable => error("Error occurred", e)
}
}
} finally {
debug("Closing server socket and selector.")
swallowError(serverChannel.close())
swallowError(nioSelector.close())
shutdownComplete()
}
}
都这儿其实Acceptor的任务就完成了。
- 总结 -
到目前为止,我们看到Kafka启动的时候会启动一个Acceptor线程,Acceptor线程启动以后会初始化一个ServerSocketChannel。 然后这个ServerSocketChannel会注册绑定到Selector上,接着一个死循环不断检查注册在Selector上的事件,如果发现是连接请求过来了,那么Acceptor调用了accept去处理这个连接,但是具体怎么处理这个连接,我们下一讲继续剖析。 大家加油!!![1e4c7b6f578bcb01c47a862d787fa6b9.png](https://i-blog.csdnimg.cn/blog_migrate/0394df153de6fc218709c099d2bf534e.jpeg)