kafka-Demo

依赖

<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
<version>2.0.0</version>
</dependency>

发送端代码

public class MyKafkaProducer extends Thread {

    //producer api
    KafkaProducer<Integer,String> producer;
    String topic;  //主题

    public MyKafkaProducer(String topic) {
        Properties properties=new Properties();
        properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG,"127.0.0.1:9092");
        properties.put(ProducerConfig.CLIENT_ID_CONFIG,"test001");//指定客户端id
//        properties.put(ProducerConfig.PARTITIONER_CLASS_CONFIG,"com.gupaoedu.kafka.MyPartition");
        properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, IntegerSerializer.class.getName());
        properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
        //连接的字符串
        //通过工厂
        //new
        producer=new org.apache.kafka.clients.producer.KafkaProducer<Integer, String>(properties);
        this.topic = topic;
    }

    @Override
    //同步发送
    public void run() {
        int num=0;
        while (num<100){
            String msg="当前发送次数为："+num;
            try {
                RecordMetadata recordMetadata = producer.send(new ProducerRecord<>(topic, msg)).get();
               
                System.out.println(recordMetadata.topic()+"-------"+recordMetadata.partition()+"----"+recordMetadata.hasOffset());
                num++;
                TimeUnit.SECONDS.sleep(2);
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    }

    //异步发送
public void run() {
        int num=0;
        while(num<50){
            String msg="pratice test message:"+num;
            try {
                producer.send(new ProducerRecord<>(topic, msg), new Callback() {
                    @Override
                    public void onCompletion(RecordMetadata recordMetadata,
                                             Exception e) {
                        System.out.println("callback:
                                "+recordMetadata.offset()+"->"+recordMetadata.partition());
                    }
                });
                TimeUnit.SECONDS.sleep(2);
                num++;
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }
    }

    public static void main(String[] args) {
        new MyKafkaProducer("test").start();

    }
}

异步发送

kafka对于消息的发送，可以支持同步和异步，前面演示的案例中，我们是基于同步发送消息。同步会 需要阻塞，而异步不需要等待阻塞的过程。 从本质上来说，kafka都是采用异步的方式来发送消息到broker，但是kafka并不是每次发送消息都会直 接发送到broker上，而是把消息放到了一个发送队列中，然后通过一个后台线程不断从队列取出消息进 行发送，发送成功后会触发callback。kafka客户端会积累一定量的消息统一组装成一个批量消息发送出 去，触发条件是前面提到的batch.size和linger.ms 而同步发送的方法，无非就是通过future.get()来等待消息的发送返回结果，但是这种方法会严重影响消 息发送的性能。 

接收端代码

public class MyKafkaConsumer extends Thread{
    KafkaConsumer<Integer,String> consumer;
    String topic;

    public MyKafkaConsumer(String topic) {
        Properties properties=new Properties();
//        properties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG,"192.168.13.102:9092,192.168.13.103:9092,192.168.13.104:9092");
        properties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG,"127.0.0.1:9092");
        properties.put(ConsumerConfig.CLIENT_ID_CONFIG,"001");
        properties.put(ConsumerConfig.GROUP_ID_CONFIG,"gid3");//消费者所属消费组的唯一标识,
        properties.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG,"30000");//超时时间
        properties.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG,"1000"); //自动提交(批量确认)
        properties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, IntegerDeserializer.class.getName());
        properties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
        //一个新的group的消费者去消费一个topic
        properties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG,"earliest"); //这个属性. 它能够消费原来发布的数据，默认七天以前
        consumer=new org.apache.kafka.clients.consumer.KafkaConsumer<Integer, String>(properties);
        this.topic = topic;
    }

    @Override
    public void run() {
        consumer.subscribe(Collections.singleton(this.topic));
        while(true){
            ConsumerRecords<Integer,String> consumerRecords=consumer.poll(Duration.ofSeconds(1));
            consumerRecords.forEach(record->{
               
                System.out.println(record.key()+"->"+record.value()+"->"+record.offset());
                try {
                    Thread.sleep(500);
                } catch (InterruptedException e) {
                    e.printStackTrace();
                }
            });
        }
    }

    public static void main(String[] args) {
        new MyKafkaConsumer("test").start();
    }
}

springboot集成Kafka

依赖

<dependency>
<groupId>org.springframework.kafka</groupId>
<artifactId>spring-kafka</artifactId>
<version>2.2.0.RELEASE</version>
</dependency>

生产者代码 

@Component
public class KafkaProducer {
@Autowired
private KafkaTemplate<String,String> kafkaTemplate;
public void send(){
kafkaTemplate.send("test","msgKey","msgData");
}
}

消费者代码

@Component
public class KafkaConsumer {
@KafkaListener(topics = {"test"})
public void listener(ConsumerRecord record){
Optional<?> msg=Optional.ofNullable(record.value());
if(msg.isPresent()){
System.out.println(msg.get());
}
}
}

测试

public static void main(String[] args) {
ConfigurableApplicationContext context=SpringApplication.run
(KafkaDemoApplication.class, args);
KafkaProducer kafkaProducer=context.getBean(KafkaProducer.class);
for(int i=0;i<3;i++){
kafkaProducer.send();
try {
Thread.sleep(3000);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}

配置文件application配置 

spring.kafka.producer.value-serializer=org.apache.kafka.common.serialization.StringSerializer
spring.kafka.producer.key-serializer=org.apache.kafka.common.serialization.IntegerSerializer

spring.kafka.bootstrap-servers=127.0.0.1:9092
#192.168.13.102:9092,192.168.13.103:9092,192.168.13.104:9092

spring.kafka.consumer.group-id=springboot-groupid
spring.kafka.consumer.auto-offset-reset=earliest
spring.kafka.consumer.enable-auto-commit=true

spring.kafka.consumer.key-deserializer=org.apache.kafka.common.serialization.IntegerDeserializer
spring.kafka.consumer.value-deserializer=org.apache.kafka.common.serialization.StringDeserializer