Kafka Topic Java API操作

最新推荐文章于 2025-03-31 19:21:52 发布

江湖峰哥

最新推荐文章于 2025-03-31 19:21:52 发布

阅读量1.3k

点赞数 1

分类专栏： # Producer专栏

本文链接：https://blog.csdn.net/querydata_boke/article/details/105368116

版权

Producer专栏专栏收录该内容

4 篇文章

订阅专栏

 private static AdminClient getAdminClient() {
   Properties admainProp = new Properties();
   admainProp.put("bootstrap.servers", "ip:9092");
   return AdminClient.create(admainProp);
 }
1. 创建主题
  @deprecated
 def create_topic(topicName: String): Unit = {
  try {
   val zkUtils = ZkUtils.apply("192.168.137.131:2181", 30000, 30000, JaasUtils.isZkSecurityEnabled)
   // 虽然过时了但是个人觉得是同步方法
   AdminUtils.createTopic(zkUtils, topicName, 10, 1, new Properties, RackAwareMode.Enforced)
   zkUtils.close()
   println(s"$topicName 创建成功~~~")
  } catch {
   case e: Exception =>
    println(s"$topicName 创建失败~~~")
    e.printStackTrace()
  }
 }
2. 删除主题
 @deprecated
 def delete_topic(topicName: String): Unit = {
  try {
   val zkUtils = ZkUtils.apply("192.168.137.131:2181", 30000, 30000, JaasUtils.isZkSecurityEnabled)
   // 同步方法
   AdminUtils.deleteTopic(zkUtils, topicName)
   zkUtils.close()
   println(s"$topicName 删除成功~~~")
  } catch {
   case e: Exception =>
    println(s"$topicName 删除失败~~~~")
    e.printStackTrace()
  }
 }
3. 获取top信息
 public static  Map<String, TopicDescription> SelectTopicInfo(String topicName) throws Exception {
   DescribeTopicsResult result = getAdminClient().describeTopics(Arrays.asList(topicName));
   KafkaFuture<Map<String, TopicDescription>> all = result.all();
   Map<String, TopicDescription> topicInfo =  all.get();
   // topic描述信息
   for(Map.Entry<String, TopicDescription> entry : topicInfo.entrySet()){
    String mapKey = entry.getKey();
    TopicDescription mapValue = entry.getValue();
    System.out.println(mapKey+":"+mapValue);
    }
   // 分区信息
   List<TopicPartitionInfo> mTopicPartitionInfo = all.get().get(topicName).partitions();
   System.out.println(mTopicPartitionInfo);
   return all.get();
  }
 4. 获取topic消息总数
  public static void getTopicTotalMessageCount(String topicName) throws Exception {
  // 借助Consumer来实现
  Properties props = new Properties();
  props.setProperty("bootstrap.servers", "ip:9092");
  props.setProperty("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
  props.setProperty("value.deserializer", "org.apache.kafka.common.serialization.ByteArrayDeserializer");
  props.setProperty("group.id", "testToal");
  props.setProperty("enable.auto.commit", "true");
  props.setProperty("auto.commit.interval.ms", "1000");
  KafkaConsumer<String, String> consumer = new KafkaConsumer<>(props);
  consumer.subscribe(Arrays.asList(topicStr));
  Map<String, TopicDescription> topicInfo = GetTopicInfo(topicStr);
  TopicDescription topicDescription = topicInfo.get(topicStr);
  List<TopicPartitionInfo> mTopicPartitionInfo = topicDescription.partitions();

  List<TopicPartition> partitions = new ArrayList<>();
  for (int i = 0; i < mTopicPartitionInfo.size(); ++i) {
  partitions.add(new TopicPartition(topicStr, i));
   }
  Map<TopicPartition, Long> offsets = consumer.endOffsets(partitions);
  Map<TopicPartition, Long> earlyOffsets = consumer.beginningOffsets(partitions);
  int totalCount = 0;
  for (TopicPartition tp : offsets.keySet()) {
  totalCount += (offsets.get(tp) - earlyOffsets.get(tp));
   }
  System.out.println("Total message count: " + totalCount);
  consumer.close();
  }
 5. 判断主题是否存在
 public static boolean topicExists(String topicName) {
  // 注意下面几个参数的值适当大一些,否则可能会出现连接异常
  // sessiontimeout in milliseconds
  // connectionTimeoutMs connection timeout in milliseconds
  // maxInFlightRequests
   KafkaZkClient kafkaClient = KafkaZkClient.apply("192.168.137.131:2181",false,20000,20000,40000,Time.SYSTEM,"","");
   return kafkaClient.topicExists(topicName);