在新版本Kafka中,__consumer_offsets这个topic是存放消费者偏移量的,但是该主题默认配置副本数量只有1,容易造成单点故障,我们可以动态修改(无需重启服务)副本因子,提高kafka的可靠性
修改流程
---------------------------------------------------------------------------------------------------------------------------
1、编写执行计划的json文件
1.1 动态地增加相关主题的副本数非常的简单,同样是使用kafka-reassign-partitions.sh
工具来执行,所以我们在操作之前得准备一个Json文件,里面详细地记录着每个分区的副本所在机器等信息,具体如下:
{
"version": 1,
"partitions": [
{
"topic": "__consumer_offsets",
"partition": 0,
"replicas": [
1,
6,
4
]
},
{
"topic": "__consumer_offsets",
"partition": 1,
"replicas": [
2,
7,
5
]
},
{
"topic": "__consumer_offsets",
"partition": 2,
"replicas": [
0,
6,
5
]
},
{
"topic": "__consumer_offsets",
"partition": 3,
"replicas": [
1,
7,
4
]
},
{
"topic": "__consumer_offsets",
"partition": 4,
"replicas": [
2,
6,
5
]
},
{
"topic": "__consumer_offsets",
"partition": 5,
"replicas": [
0,
4,
5
]
},
{
"topic": "__consumer_offsets",
"partition": 6,
"replicas": [
1,
7,
4
]
},
{
"topic": "__consumer_offsets",
"partition": 7,
"replicas": [
2,
6,
5
]
},
{
"topic": "__consumer_offsets",
"partition": 8,
"replicas": [
0,
4,
5
]
},
{
"topic": "__consumer_offsets",
"partition": 9,
"replicas": [
1,
7,
4
]
},
{
"topic": "__consumer_offsets",
"partition": 10,
"replicas": [
2,
6,
5
]
},
{
"topic": "__consumer_offsets",
"partition": 11,
"replicas": [
0,
7,
5
]
},
{
"topic": "__consumer_offsets",
"partition": 12,
"replicas": [
1,
6,
4
]
},
{
"topic": "__consumer_offsets",
"partition": 13,
"replicas": [
2,
7,
5
]
},
{
"topic": "__consumer_offsets",
"partition": 14,
"replicas": [
0,
6,
5
]
},
{
"topic": "__consumer_offsets",
"partition": 15,
"replicas": [
1,
7,
4
]
},
{
"topic": "__consumer_offsets",
"partition": 16,
"replicas": [
2,
6,
5
]
},
{
"topic": "__consumer_offsets",
"partition": 17,
"replicas": [
0,
4,
5
]
},
{
"topic": "__consumer_offsets",
"partition": 18,
"replicas": [
1,
7,
4
]
},
{
"topic": "__consumer_offsets",
"partition": 19,
"replicas": [
2,
6,
5
]
},
{
"topic": "__consumer_offsets",
"partition": 20,
"replicas": [
0,
4,
5
]
},
{
"topic": "__consumer_offsets",
"partition": 21,
"replicas": [
1,
7,
4
]
},
{
"topic": "__consumer_offsets",
"partition": 22,
"replicas": [
2,
6,
5
]
},
{
"topic": "__consumer_offsets",
"partition": 23,
"replicas": [
0,
4,
5
]
},
{
"topic": "__consumer_offsets",
"partition": 24,
"replicas": [
1,
7,
4
]
},
{
"topic": "__consumer_offsets",
"partition": 25,
"replicas": [
2,
6,
5
]
},
{
"topic": "__consumer_offsets",
"partition": 26,
"replicas": [
0,
4,
5
]
},
{
"topic": "__consumer_offsets",
"partition": 27,
"replicas": [
1,
7,
4
]
},
{
"topic": "__consumer_offsets",
"partition": 28,
"replicas": [
2,
6,
5
]
},
{
"topic": "__consumer_offsets",
"partition": 29,
"replicas": [
0,
4,
5
]
},
{
"topic": "__consumer_offsets",
"partition": 30,
"replicas": [
1,
2,
7
]
},
{
"topic": "__consumer_offsets",
"partition": 31,
"replicas": [
2,
6,
5
]
},
{
"topic": "__consumer_offsets",
"partition": 32,
"replicas": [
0,
4,
5
]
},
{
"topic": "__consumer_offsets",
"partition": 33,
"replicas": [
1,
7,
4
]
},
{
"topic": "__consumer_offsets",
"partition": 34,
"replicas": [
2,
6,
5
]
},
{
"topic": "__consumer_offsets",
"partition": 35,
"replicas": [
0,
4,
5
]
},
{
"topic": "__consumer_offsets",
"partition": 36,
"replicas": [
1,
7,
4
]
},
{
"topic": "__consumer_offsets",
"partition": 37,
"replicas": [
2,
6,
5
]
},
{
"topic": "__consumer_offsets",
"partition": 38,
"replicas": [
0,
4,
5
]
},
{
"topic": "__consumer_offsets",
"partition": 39,
"replicas": [
1,
7,
4
]
},
{
"topic": "__consumer_offsets",
"partition": 40,
"replicas": [
2,
6,
5
]
},
{
"topic": "__consumer_offsets",
"partition": 41,
"replicas": [
0,
4,
5
]
},
{
"topic": "__consumer_offsets",
"partition": 42,
"replicas": [
1,
7,
4
]
},
{
"topic": "__consumer_offsets",
"partition": 43,
"replicas": [
2,
6,
5
]
},
{
"topic": "__consumer_offsets",
"partition": 44,
"replicas": [
0,
4,
5
]
},
{
"topic": "__consumer_offsets",
"partition": 45,
"replicas": [
1,
7,
4
]
},
{
"topic": "__consumer_offsets",
"partition": 46,
"replicas": [
2,
6,
5
]
},
{
"topic": "__consumer_offsets",
"partition": 47,
"replicas": [
0,
4,
5
]
},
{
"topic": "__consumer_offsets",
"partition": 48,
"replicas": [
1,
7,
4
]
},
{
"topic": "__consumer_offsets",
"partition": 49,
"replicas": [
2,
6,
5
]
}
]
}
1.2 将json文件保存至linux中,并取名为replication.json(任意名称)
vim replication.json
1.3 执行Json文件
[root@VM_65_10_centos bin]# ./kafka-reassign-partitions.sh --zookeeper 10.69.65.2:2181 --reassignment-json-file ../explain/replcas.json --execute
1.4 验证执行计划结果
[root@VM_65_10_centos bin]# ./kafka-reassign-partitions.sh --zookeeper 10.69.65.2:2181 --reassignment-json-file ../explain/replcas.json --verify
1.5 查看新的副本数量
[root@VM_65_10_centos bin]# ./kafka-topics.sh --zookeeper 10.69.65.2:2181 --topic __consumer_offsets --describe