Kafka_2.11版本的配置文件详细参数示例

主配置文件

路径：/opt/kafka/config/server.properties

############################# Server Basics #############################
# 集群中的每个实例要配置属于自己的id，这里配置的是master节点
broker.id=1
############################# Socket Server Settings #############################
# 列出kafka监听的URI列表
listeners=PLAINTEXT://0.0.0.0:9092
# 对zookeeper发布的监听器
advertised.listeners=PLAINTEXT://172.16.3.2:9092
############################# Log Basics #############################
# log日志存储的目录，逗号分隔的多个路径，建议每个目录挂载到不同的磁盘上，能够提升读写性能，并且在1.1版本以及上可以支持故障转移
log.dirs=/data/kafka/data
# topic的默认分区数
num.partitions=3
# 在kafka重启恢复数据时使用的线程数，默认值为1
num.recovery.threads.per.data.dir=2
############################# Internal Topic Settings  #############################
# 默认的副本数量
default.replication.factor=3
# 默认的分区数量
num.partitions=3
############################# Log Flush Policy #############################
############################# Log Retention Policy #############################
# 定义数据的清理策略
# 设置数据的保存时间
log.retention.hours=168
# 限制在broker中保存的数据量，默认值为-1，不做限制
#log.retention.bytes=1073741824
############################# Zookeeper #############################
# 指定zookeeper的连接地址
zookeeper.connect=hostname-1:2181,hostname-2:2181,hostname-3:2181
# 等待zookeeper连接的超时时间
zookeeper.connection.timeout.ms=6000
############################# Group Coordinator Settings #############################
############################# User-defined Settings #############################
# 是否自动创建topic，目前没有需要自动创建topic的场景，设置为false，便于管理topic
auto.create.topics.enable=false 
# 当acks设置为all或者-1时，这个配置将规定消息写入节点的数量，如果没有达到指定的数量，生产者将会收到一个exception
min.insync.replicas=2
# 不允许不在ISR列表的副本节点选举为leader，否则可能导致数据丢失
unclean.leader.election.enable=false
# 默认值为true，会定期自动对leader partition进行重新选举，进行不必要的消耗。
auto.leader.rebalance.enable=false
# broker能够接受消息的最大字节数，默认值为1000012字节，在低于0.10.2的版本中需要对应修改消费者的fetch size，允许消费者能够拉取数据
message.max.bytes=5242880
# 要大于message.max.bytes的配置，否则broker之间无法同步数据
replica.fetch.max.bytes=10485760

日志配置文件

路径：/opt/kafka/config/log4j.properties

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Unspecified loggers and loggers with additivity=true output to server.log and stdout
# Note that INFO only applies to unspecified loggers, the log level of the child logger is used otherwise
log4j.rootLogger=INFO, stdout, kafkaAppender

log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n

log4j.appender.kafkaAppender=org.apache.log4j.DailyRollingFileAppender
log4j.appender.kafkaAppender.DatePattern='.'yyyy-MM-dd
log4j.appender.kafkaAppender.File=${kafka.logs.dir}/server.log
log4j.appender.kafkaAppender.layout=org.apache.log4j.PatternLayout
log4j.appender.kafkaAppender.layout.ConversionPattern=[%d] %p %m (%c)%n

log4j.appender.stateChangeAppender=org.apache.log4j.DailyRollingFileAppender
log4j.appender.stateChangeAppender.DatePattern='.'yyyy-MM-dd
log4j.appender.stateChangeAppender.File=${kafka.logs.dir}/state-change.log
log4j.appender.stateChangeAppender.layout=org.apache.log4j.PatternLayout
log4j.appender.stateChangeAppender.layout.ConversionPattern=[%d] %p %m (%c)%n

log4j.appender.requestAppender=org.apache.log4j.DailyRollingFileAppender
log4j.appender.requestAppender.DatePattern='.'yyyy-MM-dd
log4j.appender.requestAppender.File=${kafka.logs.dir}/kafka-request.log
log4j.appender.requestAppender.layout=org.apache.log4j.PatternLayout
log4j.appender.requestAppender.layout.ConversionPattern=[%d] %p %m (%c)%n

log4j.appender.cleanerAppender=org.apache.log4j.DailyRollingFileAppender
log4j.appender.cleanerAppender.DatePattern='.'yyyy-MM-dd
log4j.appender.cleanerAppender.File=${kafka.logs.dir}/log-cleaner.log
log4j.appender.cleanerAppender.layout=org.apache.log4j.PatternLayout
log4j.appender.cleanerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n

log4j.appender.controllerAppender=org.apache.log4j.DailyRollingFileAppender
log4j.appender.controllerAppender.DatePattern='.'yyyy-MM-dd
log4j.appender.controllerAppender.File=${kafka.logs.dir}/controller.log
log4j.appender.controllerAppender.layout=org.apache.log4j.PatternLayout
log4j.appender.controllerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n

log4j.appender.authorizerAppender=org.apache.log4j.DailyRollingFileAppender
log4j.appender.authorizerAppender.DatePattern='.'yyyy-MM-dd
log4j.appender.authorizerAppender.File=${kafka.logs.dir}/kafka-authorizer.log
log4j.appender.authorizerAppender.layout=org.apache.log4j.PatternLayout
log4j.appender.authorizerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n

# Change the line below to adjust ZK client logging
log4j.logger.org.apache.zookeeper=INFO

# Change the two lines below to adjust the general broker logging level (output to server.log and stdout)
log4j.logger.kafka=INFO
log4j.logger.org.apache.kafka=INFO

# Change to DEBUG or TRACE to enable request logging
log4j.logger.kafka.request.logger=WARN, requestAppender
log4j.additivity.kafka.request.logger=false

# Uncomment the lines below and change log4j.logger.kafka.network.RequestChannel$ to TRACE for additional output
# related to the handling of requests
#log4j.logger.kafka.network.Processor=TRACE, requestAppender
#log4j.logger.kafka.server.KafkaApis=TRACE, requestAppender
#log4j.additivity.kafka.server.KafkaApis=false
log4j.logger.kafka.network.RequestChannel$=WARN, requestAppender
log4j.additivity.kafka.network.RequestChannel$=false

log4j.logger.kafka.controller=TRACE, controllerAppender
log4j.additivity.kafka.controller=false

log4j.logger.kafka.log.LogCleaner=INFO, cleanerAppender
log4j.additivity.kafka.log.LogCleaner=false

log4j.logger.state.change.logger=TRACE, stateChangeAppender
log4j.additivity.state.change.logger=false

# Access denials are logged at INFO level, change to DEBUG to also log allowed accesses
log4j.logger.kafka.authorizer.logger=INFO, authorizerAppender
log4j.additivity.kafka.authorizer.logger=false

JVM参数配置

路径：/opt/kafka/config/java.env

KAFKA_HEAP_OPTS="-Xms4G -Xmx4G"