【kafka】--- kafka的java客户端代码

一、首先需要maven导入kafka所依赖的Jar包

<dependency>
      <groupId>org.apache.kafka</groupId>
      <artifactId>kafka-clients</artifactId>
      <version>2.1.0</version>
</dependency>

二、创建配置类，创建topic、生产者及消费者

package com.derry;

import org.apache.kafka.clients.admin.AdminClient;
import org.apache.kafka.clients.admin.AdminClientConfig;
import org.apache.kafka.clients.admin.NewTopic;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.common.serialization.StringDeserializer;
import org.apache.kafka.common.serialization.StringSerializer;

import java.io.IOException;
import java.time.Duration;
import java.util.*;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;

public class OperationKafka {

    public static void main(String[] args) throws ExecutionException, InterruptedException {
       createTopic();
       
       producer();
        
       consumer();  

    }
    
    //创建topic
    private static void createTopic() throws InterruptedException, ExecutionException {

        Properties config = new Properties();
        //指定kafka的ip及端口号
        config.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG,"ip:9092");
        config.put(AdminClientConfig.CLIENT_ID_CONFIG,"adminClient1");
        AdminClient adminClient = AdminClient.create(config);
        //创建名为lrdTest的topic及分区数为4
        adminClient.createTopics(Arrays.asList(new NewTopic("lrdTest", 4, (short) 1)));
        Set<String> topics = adminClient.listTopics().names().get();
        System.out.println("-------------->"+topics);
    }
    
    //创建消费者
    private static void consumer() {
        try {
            Properties config = new Properties();
            config.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG,"ip:9092");
            config.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
            config.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG,StringDeserializer.class.getName());       //指定group_Id
            config.put(ConsumerConfig.GROUP_ID_CONFIG,"consumer1");
            //是否自动提交
            config.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG,"true");
            //自动提交的时长
            config.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG,"100");



            KafkaConsumer consumer = new KafkaConsumer(config);
            consumer.subscribe(Arrays.asList("lrdtest"));
            ConsumerRecords<String,String> consumerRecords = consumer.poll(Duration.ofDays(1));

            for (ConsumerRecord<String,String> record : consumerRecords){
               System.out.println("recive key " + record.key()+",recive value:" + record.value());
            }

        }catch (Exception e){
            e.printStackTrace();
        }
    }
    
    //创建生产者
    private static void producer() {
        try {
            Map<String, Object> configs = new HashMap<>();
            //配置kafka的地址
            configs.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG,"ip:9092");
            //键序列化
            configs.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
            //值序列化
            configs.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
            //acks:消息的确认机制，默认值是0。
            //acks=0：如果设置为0，生产者不会等待kafka的响应。
            //acks=1：这个配置意味着kafka会把这条消息写到本地日志文件中，但是不会等待集群中其他机器的成功响应。
            //acks=all：这个配置意味着leader会等待所有的follower同步完成。这个确保消息不会丢失，除非kafka集群中所有机器挂掉。这是最强的可用性保证。
            configs.put(ProducerConfig.ACKS_CONFIG, "all");
            //客户端ID,用来确认是哪个应用发出了该请求
            configs.put(ProducerConfig.CLIENT_ID_CONFIG,"productor1");
            //配置为大于0的值的话，客户端会在消息发送失败时重新发送。
            configs.put(ProducerConfig.RETRIES_CONFIG,"0");
            //当多条消息需要发送到同一个分区时，生产者会尝试合并网络请求。这会提高client和生产者的效率。
            configs.put(ProducerConfig.BATCH_SIZE_CONFIG,"1");
            configs.put(ProducerConfig.LINGER_MS_CONFIG,"1");
            configs.put(ProducerConfig.BUFFER_MEMORY_CONFIG,"10240");
            //broker尽力实现request.required.acks需求时的等待时间，否则会发送错误到客户端
            configs.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG,"20000");
            configs.put(ProducerConfig.DELIVERY_TIMEOUT_MS_CONFIG,"30000");
            KafkaProducer producer = new KafkaProducer(configs);
            for (int i=0;i<10;i++) {
                String result = UUID.randomUUID().toString();
                String topic = "lrdtest";
                producer.send(new ProducerRecord(topic,result));
                System.out.println("send"+i+"to" + topic );
                TimeUnit.SECONDS.sleep(2);
            }
            producer.close();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

}

三、一些常用的脚本命令

在kafka/bin目录下执行一下操作

#创建一个名为test的topic，分区数为2，副本数为2
kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 2 --partitions 2 --topic test

#查看该zookeeper下已创建多的topic list
kafka-topics.sh --list --zookeeper localhost:2181

#创建一个消费topic:test的一个消费者
kafka-console-producer.sh --broker-list localhost:9092 --topic test

#创建一个消费者，消费topic:test
kafka-console-consumer.sh --zookeeper localhost:9092 --topic test --from-beginning