Kafka Java API

最新推荐文章于 2021-08-19 11:23:44 发布

套码的汉子

最新推荐文章于 2021-08-19 11:23:44 发布

阅读量417

点赞数 1

分类专栏： Kafka 文章标签： kafka

本文链接：https://blog.csdn.net/qq_43502858/article/details/96741357

版权

本文详细介绍了如何使用Kafka的Java API进行Topic管理，包括创建、列举和删除Topic，以及修改分区数量。同时，展示了如何创建生产者向指定主题推送消息，以及创建消费者从不同方式拉取数据。此外，还涵盖了自定义分区器的实现。

摘要由CSDN通过智能技术生成

文章目录

Kafka Java API

Kafka Java API

准备工作

（1）创建Maven项目，导入相关依赖

    <dependencies>
        <dependency>
            <groupId>org.apache.kafka</groupId>
            <artifactId>kafka_2.11</artifactId>
            <version>0.10.0.1</version>
        </dependency>
        <dependency>
            <groupId>org.apache.kafka</groupId>
            <artifactId>kafka-clients</artifactId>
            <version>2.0.0</version>
        </dependency>
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>4.11</version>
            <scope>test</scope>
        </dependency>
    </dependencies>

（2）将kafka config目录下的server.properties,consumer.properties,producer.properties,log4j.properties拷贝放到项目的resources目录下

在这里插入图片描述

server.properties

broker.id=0
listeners=PLAINTEXT://pseudo01:9092
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=/opt/software/kafka/kafka-0.10/logs
num.partitions=1
num.recovery.threads.per.data.dir=1
log.retention.hours=168
log.segment.bytes=1048576
log.retention.check.interval.ms=300000
zookeeper.connect=pseudo01:2181
zookeeper.connection.timeout.ms=6000
zookeeper.session.timeout.ms=6000

producer.properties

bootstrap.servers=pseudo01:9092
compression.type=none
key.serializer=org.apache.kafka.common.serialization.StringSerializer
value.serializer=org.apache.kafka.common.serialization.StringSerializer

consumer.properties

zookeeper.connect=192.168.142.92:2181
bootstrap.servers=192.168.142.92:9092
zookeeper.connection.timeout.ms=6000
group.id=my-consumer-group
key.deserializer=org.apache.kafka.common.serialization.StringDeserializer
value.deserializer=org.apache.kafka.common.serialization.StringDeserializer

1.使用Java API操作Topic

TopicHandler.java

import kafka.admin.AdminUtils;
import kafka.admin.RackAwareMode;
import kafka.cluster.Broker;
import kafka.common.TopicAndPartition;
import kafka.utils.ZkUtils;
import org.apache.kafka.common.security.JaasUtils;
import scala.collection.JavaConversions;
import scala.collection.Seq;
import scala.collection.Set;

import java.io.IOException;
import java.io.InputStream;
import java.util.Collection;
import java.util.List;
import java.util.Properties;

public class TopicHandler {
   
    private static Properties properties;
    private static String zkUrl;
    private static Integer sessionTimeOut;
    private static Integer connectionTimeOut;

    static {
   
        InputStream stream = TopicHandler.class.getClassLoader().getResourceAsStream("server.properties");
        properties = new Properties();
        try {
   
            properties.load(stream);
            zkUrl = properties.getProperty("zookeeper.connect");
            sessionTimeOut = Integer.valueOf(properties.getProperty("zookeeper.session.timeout.ms"));
            connectionTimeOut = Integer.valueOf(properties.getProperty("zookeeper.connection.timeout.ms"));
        } catch (IOException e) {
   
            System.out.println("读取配置文件失败！！！");
            e.printStackTrace();
        }
    }

    /**
     * 获取ZkUtils对象
     * @return
     */
    public static ZkUtils getZkUtils(){
   
        return ZkUtils.apply(zkUrl,sessionTimeOut,connectionTimeOut, JaasUtils.isZkSecurityEnabled());
    }

    /**
     * 创建topic
     * @param topic        主题名
     * @param replications 副本数
     * @param partitions   分区数
     */
    public static void createTopic(String topic,int replications,int partitions){
   
        ZkUtils zkUtils = getZkUtils();
        AdminUtils.createTopic(zkUtils,topic,partitions,replications,new Properties(), RackAwareMode.Safe$.MODULE$);
        zkUtils.close();
    }

    /**
     * 列出所有分区
     */
    public static void listTopics(){
   
        ZkUtils zkUtils = getZkUtils();
        Seq<String> topicsSeq = zkUtils.getAllTopics();
        List<String> topics = JavaConversions.seqAsJavaList(topicsSeq);
        topics.forEach(System.out::println);
        zkUtils.close();
    }

    /**
     * 删除指定topic
     * @param topic 主题名
     */
    public static void delTopic(String topic){
   
        ZkUtils zkUtils = getZkUtils();
        AdminUtils.deleteTopic(zkUtils,topic);
        zkUtils.close();
    }

    public static void listTopicPartitions(String topic