05-kafka FAQ（常见问题解答）-集群一个节点挂掉topic就不能访问的问题

1. 查看下当前_consumer_offsets的副本数

[root@kafka-01 bin]# ./kafka-topics.sh --zookeeper 10.10.239.61:2181 --topic __consumer_offsets --describe

输出如下：

Topic: __consumer_offsets       PartitionCount: 50      ReplicationFactor: 1    Configs: compression.type=producer,cleanup.policy=compact,segment.bytes=104857600
        Topic: __consumer_offsets       Partition: 0    Leader: 3       Replicas: 3     Isr: 3
        Topic: __consumer_offsets       Partition: 1    Leader: 1       Replicas: 1     Isr: 1
        Topic: __consumer_offsets       Partition: 2    Leader: 2       Replicas: 2     Isr: 2
        Topic: __consumer_offsets       Partition: 3    Leader: 3       Replicas: 3     Isr: 3
        Topic: __consumer_offsets       Partition: 4    Leader: 1       Replicas: 1     Isr: 1
       ……
        Topic: __consumer_offsets       Partition: 48   Leader: 3       Replicas: 3     Isr: 3
        Topic: __consumer_offsets       Partition: 49   Leader: 1       Replicas: 1     Isr: 1

如上图可见，都只有一个节点，或是1、或是2、或是3。

2. 动态修改

用自带的 kafka-reassign-partitions.sh工具修改

• 创建consumer_offset.json文件

名字无所谓，执行的时候调用这个文件即可。

{
    "version": 1, 
    "partitions": [
        {
            "topic": "__consumer_offsets", 
            "partition": 0, 
            "replicas": [1, 2, 3]
        },
        {
            "topic": "__consumer_offsets", 
            "partition": 1, 
            "replicas": [1, 3, 2]
        },
        {
            "topic": "__consumer_offsets", 
            "partition": 2, 
            "replicas": [2, 1, 3]
        },
        {
            "topic": "__consumer_offsets", 
            "partition": 3, 
            "replicas": [2, 3, 1]
        },
        {
            "topic": "__consumer_offsets", 
            "partition": 4, 
            "replicas": [3, 1, 2]
        },
        {
            "topic": "__consumer_offsets", 
            "partition": 5, 
            "replicas": [3, 2, 1]
        },
        {
            "topic": "__consumer_offsets", 
            "partition": 6, 
            "replicas": [1, 2, 3]
        },
        {
            "topic": "__consumer_offsets", 
            "partition": 7, 
            "replicas": [1, 3, 2]
        },
        {
            "topic": "__consumer_offsets", 
            "partition": 8, 
            "replicas": [2, 1, 3]
        },
        {
            "topic": "__consumer_offsets", 
            "partition": 9, 
            "replicas": [2, 3, 1]
        },
        {
            "topic": "__consumer_offsets", 
            "partition": 10, 
            "replicas": [3, 1, 2]
        },
        {
            "topic": "__consumer_offsets", 
            "partition": 11, 
            "replicas": [3, 2, 1]
        },
        {
            "topic": "__consumer_offsets", 
            "partition": 12, 
            "replicas": [1, 2, 3]
        },
        {
            "topic": "__consumer_offsets", 
            "partition": 13, 
            "replicas": [1, 3, 2]
        },
        {
            "topic": "__consumer_offsets", 
            "partition": 14, 
            "replicas": [2, 1, 3]
        },
        {
            "topic": "__consumer_offsets", 
            "partition": 15, 
            "replicas": [2, 3, 1]
        },
        {
            "topic": "__consumer_offsets", 
            "partition": 16, 
            "replicas": [3, 1, 2]
        },
        {
            "topic": "__consumer_offsets", 
            "partition": 17, 
            "replicas": [3, 2, 1]
        },
        {
            "topic": "__consumer_offsets", 
            "partition": 18, 
            "replicas": [1, 2, 3]
        },
        {
            "topic": "__consumer_offsets", 
            "partition": 19, 
            "replicas": [1, 3, 2]
        },
        {
            "topic": "__consumer_offsets", 
            "partition": 20, 
            "replicas": [2, 1, 3]
        },
        {
            "topic": "__consumer_offsets", 
            "partition": 21, 
            "replicas": [2, 3, 1]
        },
        {
            "topic": "__consumer_offsets", 
            "partition": 22, 
            "replicas": [3, 1, 2]
        },
        {
            "topic": "__consumer_offsets", 
            "partition": 23, 
            "replicas": [3, 2, 1]
        },
        {
            "topic": "__consumer_offsets", 
            "partition": 24, 
            "replicas": [1, 2, 3]
        },
        {
            "topic": "__consumer_offsets", 
            "partition": 25, 
            "replicas": [1, 3, 2]
        },
        {
            "topic": "__consumer_offsets", 
            "partition": 26, 
            "replicas": [2, 1, 3]
        },
        {
            "topic": "__consumer_offsets", 
            "partition": 27, 
            "replicas": [2, 3, 1]
        },
        {
            "topic": "__consumer_offsets", 
            "partition": 28, 
            "replicas": [3, 1, 2]
        },
        {
            "topic": "__consumer_offsets", 
            "partition": 29, 
            "replicas": [3, 2, 1]
        },
        {
            "topic": "__consumer_offsets", 
            "partition": 30, 
            "replicas": [1, 2, 3]
        },
        {
            "topic": "__consumer_offsets", 
            "partition": 31, 
            "replicas": [1, 3, 2]
        },
        {
            "topic": "__consumer_offsets", 
            "partition": 32, 
            "replicas": [2, 1, 3]
        },
        {
            "topic": "__consumer_offsets", 
            "partition": 33, 
            "replicas": [2, 3, 1]
        },
        {
            "topic": "__consumer_offsets", 
            "partition": 34, 
            "replicas": [3, 1, 2]
        },
        {
            "topic": "__consumer_offsets", 
            "partition": 35, 
            "replicas": [3, 2, 1]
        },
        {
            "topic": "__consumer_offsets", 
            "partition": 36, 
            "replicas": [1, 2, 3]
        },
        {
            "topic": "__consumer_offsets", 
            "partition": 37, 
            "replicas": [1, 3, 2]
        },
        {
            "topic": "__consumer_offsets", 
            "partition": 38, 
            "replicas": [2, 1, 3]
        },
        {
            "topic": "__consumer_offsets", 
            "partition": 39, 
            "replicas": [2, 3, 1]
        },
        {
            "topic": "__consumer_offsets", 
            "partition": 40, 
            "replicas": [3, 1, 2]
        },
        {
            "topic": "__consumer_offsets", 
            "partition": 41, 
            "replicas": [3, 2, 1]
        },
        {
            "topic": "__consumer_offsets", 
            "partition": 42, 
            "replicas": [1, 2, 3]
        },
        {
            "topic": "__consumer_offsets", 
            "partition": 43, 
            "replicas": [1, 3, 2]
        },
        {
            "topic": "__consumer_offsets", 
            "partition": 44, 
            "replicas": [2, 1, 3]
        },
        {
            "topic": "__consumer_offsets", 
            "partition": 45, 
            "replicas": [2, 3, 1]
        },
        {
            "topic": "__consumer_offsets", 
            "partition": 46, 
            "replicas": [3, 1, 2]
        },
        {
            "topic": "__consumer_offsets", 
            "partition": 47, 
            "replicas": [3, 2, 1]
        },
        {
            "topic": "__consumer_offsets", 
            "partition": 48, 
            "replicas": [1, 2, 3]
        },
        {
            "topic": "__consumer_offsets", 
            "partition": 49, 
            "replicas": [1, 3, 2]
        }
    ]
}

• 执行修改脚本

[root@kafka-01 bin]# ./kafka-reassign-partitions.sh --zookeeper 10.10.239.61:2181 --reassignment-json-file consumer_offset.json --execute

• 查看修改结果

[root@kafka-01 bin]# ./kafka-topics.sh --zookeeper 10.10.239.61:2181 --topic __consumer_offsets --describe
./kafka-topics.sh --zookeeper 10.10.239.61:2181 --topic __consumer_offsets --describe
Topic: __consumer_offsets       PartitionCount: 50      ReplicationFactor: 3    Configs: compression.type=producer,cleanup.policy=compact,segment.bytes=104857600
        Topic: __consumer_offsets       Partition: 0    Leader: 3       Replicas: 1,2,3 Isr: 3,1,2
        Topic: __consumer_offsets       Partition: 1    Leader: 1       Replicas: 1,3,2 Isr: 1,2,3
        Topic: __consumer_offsets       Partition: 2    Leader: 2       Replicas: 2,1,3 Isr: 2,1,3
        Topic: __consumer_offsets       Partition: 3    Leader: 3       Replicas: 2,3,1 Isr: 3,2,1
        Topic: __consumer_offsets       Partition: 4    Leader: 1       Replicas: 3,1,2 Isr: 1,2,3
       ……

3. 修改服务

• 修改配置文件server.properties

############################# Internal Topic Settings  #############################
# The replication factor for the group metadata internal topics "__consumer_offsets" and "__transaction_state"
# For anything other than development testing, a value greater than 1 is recommended to ensure availability such as 3.
offsets.topic.replication.factor=3
transaction.state.log.replication.factor=3
transaction.state.log.min.isr=3

• 重启kafka
  （略）

---