1. 基于java API 开发
首先新建pom工程,除了kafka,还需要引入序列化的坐标:
<dependencies>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
<version>2.6.0</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.curator/curator-client -->
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-framework</artifactId>
<version>4.2.0</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.9.5</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.google.protobuf/protobuf-java -->
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
<version>3.11.4</version>
</dependency>
</dependencies>
然后创建生产者:
public class SimpleProducer {
public static void main(String[] args) {
Properties pros=new Properties();
//pros.put("bootstrap.servers","192.168.44.161:9093,192.168.44.161:9094,192.168.44.161:9095");
pros.put("bootstrap.servers","192.168.3.128:9092");
pros.put("key.serializer","org.apache.kafka.common.serialization.StringSerializer");
pros.put("value.serializer","org.apache.kafka.common.serialization.StringSerializer");
// 0 发出去就确认 | 1 leader 落盘就确认| all(-1) 所有Follower同步完才确认
pros.put("acks","1");
// 异常自动重试次数
pros.put("retries",3);
// 多少条数据发送一次,默认16K
pros.put("batch.size",16384);
// 批量发送的等待时间
pros.put("linger.ms",5);
// 客户端缓冲区大小,默认32M,满了也会触发消息发送
pros.put("buffer.memory",33554432);
// 获取元数据时生产者的阻塞时间,超时后抛出异常
pros.put("max.block.ms",3000);
// 创建Sender线程
Producer<String,String> producer = new KafkaProducer<String,String>(pros);
for (int i =0 ;i<10;i++) {
producer.send(new ProducerRecord<String,String>("mytopic",Integer.toString(i),Integer.toString(i)));
System.out.println("发送:"+i);
}
//producer.send(new ProducerRecord<String,String>("mytopic","1","1"));
//producer.send(new ProducerRecord<String,String>("mytopic","2","2"));
producer.close();
}
}
再创建消费者:
public class SimpleConsumer {
public static void main(String[] args) {
Properties props= new Properties();
//props.put("bootstrap.servers","192.168.44.161:9093,192.168.44.161:9094,192.168.44.161:9095");
props.put("bootstrap.servers","192.168.3.128:9092");
props.put("group.id","gp-test-group");
// 是否自动提交偏移量,只有commit之后才更新消费组的 offset
props.put("enable.auto.commit","true");
// 消费者自动提交的间隔
props.put("auto.commit.interval.ms","1000");
// 从最早的数据开始消费 earliest | latest | none
props.put("auto.offset.reset","earliest");
props.put("key.deserializer","org.apache.kafka.common.serialization.StringDeserializer");
props.put("value.deserializer","org.apache.kafka.common.serialization.StringDeserializer");
KafkaConsumer<String,String> consumer=new KafkaConsumer<String, String>(props);
// 订阅topic
consumer.subscribe(Arrays.asList("mytopic"));
try {
while (true){
ConsumerRecords<String,String> records=consumer.poll(Duration.ofMillis(1000));
for (ConsumerRecord<String,String> record:records){
System.out.printf("offset = %d ,key =%s, value= %s, partition= %s%n" ,record.offset(),record.key(),record.value(),record.partition());
}
}
}finally {
consumer.close();
}
}
}
先启动生产者,再启动消费者,可以看到消费者的打印为:
2.基于SpringBoot的使用方法
基于SpringBoot的最大好处之一就是将配置相关的内容都写在配置文件里。另外还可以使用KafkaTemplate来帮助我们完成大量基础工作。
我们创建一个pom工程,然后创建application.properties文件:
server.port=7271
#spring.kafka.bootstrap-servers=192.168.44.161:9093,192.168.44.161:9094,192.168.44.161:9095
spring.kafka.bootstrap-servers=192.168.3.128:9092
# producer
spring.kafka.producer.retries=1
spring.kafka.producer.batch-size=16384
spring.kafka.producer.buffer-memory=33554432
spring.kafka.producer.acks=all
spring.kafka.producer.properties.linger.ms=5
spring.kafka.producer.key-serializer=org.apache.kafka.common.serialization.StringSerializer
spring.kafka.producer.value-serializer=org.apache.kafka.common.serialization.StringSerializer
# consumer
spring.kafka.consumer.auto-offset-reset=earliest
spring.kafka.consumer.enable-auto-commit=true
spring.kafka.consumer.auto-commit-interval=1000
spring.kafka.consumer.key-deserializer=org.apache.kafka.common.serialization.StringDeserializer
spring.kafka.consumer.value-deserializer=org.apache.kafka.common.serialization.StringDeserializer
生产者:
@Component
public class KafkaProducer {
@Autowired
private KafkaTemplate<String,Object> kafkaTemplate;
public String send(@RequestParam String msg){
kafkaTemplate.send("springboottopic", msg);
return "ok";
}
}
监听:
@Component
public class ConsumerListener {
@KafkaListener(topics = "springboottopic",groupId = "springboottopic-group")
public void onMessage(String msg){
System.out.println("----收到消息:"+msg+"----");
}
}
我们再创建一个测试方法:
@SpringBootTest
class KafkaTests {
@Autowired
KafkaProducer producer;
// 消费者:先启动 kafkaApp
@Test
void testSendMsg() {
long time = System.currentTimeMillis();
System.out.println("----"+time +",已经发出----");
producer.send("ttttt, " +time);
}
}
这样启动main方法就ok了