Kafka安装与使用

一、安装

• 解压kafka安装文件.tar.gz

• 在/opt/module/kafka 目录下创建logs文件夹 mkdir logs

• 修改配置文件 cd config/ vi server.properties

• server.properties中

  • 启动删除topic功能 delete.topic.enable=true
  • kafka存放路径（数据与日志共同存放）log.dirs=/opt/module/kafka/logs
  • 配置zookeeper集群地址 例：zookeeper.connect=hadoop102:2181（中间以逗号分隔）

• 环境变量配置

  • vi /etc/profile

  • #KAFKA_HOME

    export KAFKA_HOME=opt/module/kafka

    export PATH=$PATH:$KAFKA_HOME/bin

    source /etc/profile

• 分发安装包 xsync kafka/

二、启动

bin/kafka-server-start.sh -daemon config/server.properties

三、增删查topic

#查询
bin/kafka-topics.sh --list --zookeeper hadoop102:8181
#增加
bin/kakfka-topics.sh --create --zookeeper hadoop102:8181 --topic first --partitions 2 --replication-factor 2
#删除
bin/kafka-topics.sh --delete --zookeeper hadoop102:8181 --topic first
#查询详情
bin/kafka-topics.sh --describe --topic first --zookeeper hadoop102:2181

四、生产者Api

private void getKafkaProducer(){
    Properties properties = new Properties();
    //指定连接的kafka集群
    properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG,"hadoop102:9002");
    //ack 应答级别
    properties.put(ProducerConfig.ACKS_CONFIG,"all");
    //重试次数
    properties.put(ProducerConfig.RETRIES_CONFIG,1);
    //批次大小 16k
    properties.put(ProducerConfig.BATCH_SIZE_CONFIG,16384);
    //等待时间
    properties.put(ProducerConfig.LINGER_MS_CONFIG,1);
    //recordAccumulator缓冲区大小
    properties.put(ProducerConfig.BUFFER_MEMORY_CONFIG,33554432);
    //key序列化
    properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serialization.StringSerializer");
    //value序列化
    properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serialization.StringSerializer");
    //创建生产者对象
    KafkaProducer producer = new KafkaProducer<String,String>(properties);
    //发送数据
    for (int i = 0; i < 10; i++) {
        producer.send(new ProducerRecord("number",100+i));
    }
    //关闭资源
    producer.close();
}

五、消费者Api

public void getKafkaConsumer(){
        Properties properties = new Properties();
        //连接集群
        properties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG,"hadoop102:9002");
        //开启自动提交
        properties.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG,true);
        //自动提交延迟
        properties.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG,"1000");
        //反序列化
        properties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serialization.StringDeserializer");
        properties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serialization.StringDeserializer");
        //消费组
        properties.put(ConsumerConfig.GROUP_ID_CONFIG,"data");
        //重置消费组的offset
        properties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG,"earliest");
        KafkaConsumer consumer = new KafkaConsumer<String,String>(properties);
        //订阅主题
        consumer.subscribe(Arrays.asList("f","s"));

        while (true){
            ConsumerRecords<String,String> poll = consumer.poll(100);
            poll.forEach(p->{
                System.out.println(p.key()+p.value());
            });
        }
    }