Kafka学习笔记（四）·主题Topic的管理

四、主题

1、管理

1、创建主题

[root@k8s ~]# bin/kafka-topics.sh --zookeeper localhost:2181 --create --topic hello --partitions 2 --replication-factor 1
# localhost:2181	zookeeper所在的ip，zookeeper必穿参数，多个zookeeper用逗号分开
# partitions 用于设置主题分区数，每个线程处理一个分区数据
# replication-factor用于设置主题副本数，每个副本分布再不同节点，不能超过总结点数。如果只有一个节点，但是创		建时指定副本数为2，就会报错

• 查看topic元数据信息的方法

  topic元数据信息保存在zookeeper节点中

# 连接zk client
[root@k8s ~]# bin/zkCli.sh -server localhost:2181

# 查看元数据信息
[root@k8s ~]# get /brokers/topics/hello
[root@k8s ~]#

查询结果效果如下

2、查看主题

# 查看所有主题
bin/kafka-topics.sh --list --zookeeper localhost:2181
hello	#显示结果

# 查看某个特定主题信息，不指定topic则查询所有。通过  --describe
[root@k8s ~]# bin/kafka-topics.sh --zookeeper localhost:2181 --describe --topic hello
Topic:hello		PartitionCount:2	ReplicationFactor:1		configs:
Topic:hello		Partition: 0		Leader: 0	Replicas: 0		Isr: 0
Topic:hello		Partition: 1		Leader: 0	Replicas: 0		Isr: 0

# 查看正在同步的主题
#  通过 --describe和under-replicated-partitions命令组合查看under-replacation状态

3、修改主题

# 增加配置
[root@k8s ~]# bin/kafka-topic.sh --alter --zookeeper localhost:2181 --topic hello --config flush.message=1

# 删除配置
[root@k8s ~]# bin/kafka-topics.sh --alter --zookeeper localhost:2181 --topic hello --delete-config flush.messages

4、删除主题

• 若delete.topic.enable=true ，则直接彻底删除该topic
• 若delete.topic.enable=false
  • 如果当前topic没有使用过即没有传输过信息，可以彻底删除。
  • 如果当前topic有使用过即传输过信息，并没有真正删除topic只是把这个topic标记为删除(marked for deletion)，重启kafka server后删除

[root@k8s ~]# bin/kafka-topics.sh --delete --zookeeper localhost:2181 --topic hello

# 标记为删除
[root@k8s ~]# bin/kafka-topics.sh --list --zookeeper localhost:2181
hello - marked for deletion

2、增加分区

# 增加分区数
[root@k8s ~]# bin/kafka-topics.sh --alter --zookeeper localhost:2181 --topic hello --partitions 3
WARNING: If partitions are increased for a topic that has a key,the partition logic or ordering of the messages will be affected
Adding partitions succeeded!

# 修改分区数时，仅能增加分区个数。若是用其减少partition个数，则会报如下错误：
[root@k8s ~]# bin/kafka-topics.sh --alter --zookeeper localhost:2181 --topic hello --partitions 2
WARNING: If partitions are increased for a topic that has a key,the partition logic or ordering of the messages will be affected
Error while executing topic command:The number of partitions for a topic can only be increased. Topic hello currently has 3 partitions,2 would not be an increase.
[2020-12-07 09:37:21,478] ERROR org.apache.kafka.common.errors.InvalidPartitionsException:The number of partitions for a topic can only be increases.Topic hello currently has 3 partitions,2 would not be an increase

3、KafkaAdminClient应用

我们都习惯使用kafka中bin目录下的脚本工具来管理查看kafka，但是有些时候需要将某些管理查看的功能集成到系统（比如kafka manager）中，那么就需要调用一些API来直接操作kafka了。

public class KafkaAdminConfigOperation{
    public static void main(String[] args) throws Exception{
		//describeTopicConfig();
        //alterTopicConfig();
        addTopicPartitions();
    }
    
    public static void describeTopicConfig() throws Exception{
        String brokerList="localhost:9092";
        String topic="hello";
        
        Properties props=new Properties();
        props.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG,brokerList);
        props.put(AdminClientConfig.REQUEST_TIMEOUT_MS_CONFIG,30000);
        AdminClient client=AdminClient.create(props);
        
        ConfigResource resource=new ConfigResource(ConfigResource.Type.TOPIPC,topic);
        DescribeConfigsResult result=
            		client.describeConfigs(Collections.signleton(resource));
        Config config=result.all().get().get(resource);
        System.out.println(config);
        client.close();
    }
    
    public static void alterTopicConfig() throws Exception{
        String brokerList="localhost:9092";
        String topic="hello";
        
        Properties props=new Properties();
        props.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG,brokerList);
        props.put(AdminClientConfig.REQUEST_TIMEOUT_MS_CONFIG,30000);
        AdminClient client=AdminClient.create(props);
        
        ConfigResource resource=new ConfigResource(ConfigResource.Type.TOPIPC,topic);
        ConfigEntry entry=new ConfigEntry("cleanup.policy","compact");
        Config config=new Config(Collections.signleton(entry));
        Map<ConfigResource,Config> configs=new HashMap<>();
        configs.put(resource,config);
        AlterConfigResult result=client.alterConfigs(conofigs);
        result.all().get();
        
        client.close();
    }
    
    public static void addTopicPartitions() throws Exception{
        String brokerList="localhost:9092";
        String topic="hello";
        
        Properties props=new Properties();
        props.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG,brokerList);
        props.put(AdminClientConfig.REQUEST_TIMEOUT_MS_CONFIG,30000);
        AdminClient client=AdminClient.create(props);
        
        NewPartitions newPartitions=NewPartitions.increaseTo(3);
        Map<String,NewPartitions> new PartitionsMap=new HashMap<>();
        newPartitionsMap.put(topic,newPartitions);
        CreatePartitionsResult result=client.createPartitions(newPartitionsMap);
        result.all().get();
        
        client.close();
    }
}