Kafka API实战
依赖
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
<version>2.2.0</version>
</dependency>
<!-- https://mvnrepository.com/artifact/log4j/log4j -->
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<version>1.2.17</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.slf4j/slf4j-api -->
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.7.25</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.slf4j/slf4j-log4j12 -->
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>1.7.25</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.8.1</version>
</dependency>
log4j配置
log4j.rootLogger = info,console
log4j.appender.console = org.apache.log4j.ConsoleAppender
log4j.appender.console.Target = System.out
log4j.appender.console.layout = org.apache.log4j.PatternLayout
log4j.appender.console.layout.ConversionPattern = %p %d{yyyy-MM-dd HH:mm:ss} %c - %m%n
生产者
package com. baizhi. jsy. kafkaapi;
import org. apache. kafka. clients. producer. KafkaProducer;
import org. apache. kafka. clients. producer. ProducerConfig;
import org. apache. kafka. clients. producer. ProducerRecord;
import org. apache. kafka. common. serialization. StringSerializer;
import java. text. DecimalFormat;
import java. util. Properties;
public class ProductKafka {
public static void main ( String[ ] args) {
Properties properties = new Properties ( ) ;
properties. put ( ProducerConfig. BOOTSTRAP_SERVERS_CONFIG, "Centos:9092" ) ;
properties. put ( ProducerConfig. KEY_SERIALIZER_CLASS_CONFIG, StringSerializer. class . getName ( ) ) ;
properties. put ( ProducerConfig. VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer. class . getName ( ) ) ;
properties. put ( ProducerConfig. BATCH_SIZE_CONFIG, 1024 * 1024 ) ;
properties. put ( ProducerConfig. LINGER_MS_CONFIG, 500 ) ;
KafkaProducer< String, String> kafkaProducer = new KafkaProducer < String, String> ( properties) ;
for ( int i= 0 ; i< 10 ; i++ ) {
DecimalFormat decimalFormat = new DecimalFormat ( "00" ) ;
String format = decimalFormat. format ( i) ;
ProducerRecord< String, String> record = new ProducerRecord < > ( "topic01" , "key" + format, "value" + format) ;
kafkaProducer. send ( record) ;
}
kafkaProducer. flush ( ) ;
kafkaProducer. close ( ) ;
}
}
消费者
package com. baizhi. jsy. kafkaapi;
import org. apache. kafka. clients. consumer. ConsumerConfig;
import org. apache. kafka. clients. consumer. ConsumerRecord;
import org. apache. kafka. clients. consumer. ConsumerRecords;
import org. apache. kafka. clients. consumer. KafkaConsumer;
import org. apache. kafka. common. serialization. StringDeserializer;
import java. time. Duration;
import java. util. Arrays;
import java. util. Iterator;
import java. util. Properties;
import java. util. regex. Pattern;
public class ConsumerKafka {
public static void main ( String[ ] args) {
Properties properties = new Properties ( ) ;
properties. put ( ConsumerConfig. BOOTSTRAP_SERVERS_CONFIG, "Centos:9092" ) ;
properties. put ( ConsumerConfig. KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer. class . getName ( ) ) ; properties. put ( ConsumerConfig. VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer. class . getName ( ) ) ;
properties. put ( ConsumerConfig. AUTO_OFFSET_RESET_CONFIG, "earliest" ) ;
properties. put ( ConsumerConfig. GROUP_ID_CONFIG, "group02" ) ;
KafkaConsumer< String, String> kafkaConsumer = new KafkaConsumer < String, String> ( properties) ;
kafkaConsumer. subscribe ( Arrays. asList ( "topic01" ) ) ;
try {
while ( true ) {
ConsumerRecords< String, String> consumerRecords = kafkaConsumer. poll ( Duration. ofSeconds ( 1 ) ) ;
if ( ! consumerRecords. isEmpty ( ) ) {
Iterator< ConsumerRecord< String, String> > iterator = consumerRecords. iterator ( ) ;
while ( iterator. hasNext ( ) ) {
ConsumerRecord< String, String> next = iterator. next ( ) ;
String topic = next. topic ( ) ;
System. out. println ( "topic = " + topic) ;
String key = next. key ( ) ;
System. out. println ( "key = " + key) ;
String value = next. value ( ) ;
System. out. println ( "value = " + value) ;
long offset = next. offset ( ) ;
System. out. println ( "offset = " + offset) ;
int partition = next. partition ( ) ;
System. out. println ( "partition = " + partition) ;
long timestamp = next. timestamp ( ) ;
System. out. println ( "timestamp = " + timestamp) ;
System. out. println ( ) ;
}
}
}
} catch ( Exception e) {
e. printStackTrace ( ) ;
} finally {
kafkaConsumer. close ( ) ;
}
}
}
序列化的案例
创建user实体类
package com. baizhi. jsy. serializer;
import java. io. Serializable;
import java. util. Date;
public class User implements Serializable {
private String id;
private String name;
private Date date;
public User ( ) {
}
public User ( String id, String name, Date date) {
this . id = id;
this . name = name;
this . date = date;
}
@Override
public String toString ( ) {
return "User{" +
"id='" + id + '\'' +
", name='" + name + '\'' +
", date=" + date +
'}' ;
}
public String getId ( ) {
return id;
}
public void setId ( String id) {
this . id = id;
}
public String getName ( ) {
return name;
}
public void setName ( String name) {
this . name = name;
}
public Date getDate ( ) {
return date;
}
public void setDate ( Date date) {
this . date = date;
}
}
user序列化
package com. baizhi. jsy. serializer;
import org. apache. commons. lang3. SerializationUtils;
import org. apache. kafka. common. serialization. Serializer;
import java. io. Serializable;
import java. util. Map;
public class Userserializer implements Serializer < Object> {
@Override
public void configure ( Map< String, ? > map, boolean b) { }
@Override
public byte [ ] serialize ( String topic, Object data) {
return SerializationUtils. serialize ( ( Serializable) data) ;
}
@Override
public void close ( ) { }
}
user反序列化
package com. baizhi. jsy. serializer;
import org. apache. commons. lang3. SerializationUtils;
import org. apache. kafka. common. serialization. Deserializer;
import java. util. Map;
public class UserDeserializer implements Deserializer < Object> {
@Override
public void configure ( Map< String, ? > map, boolean b) { }
@Override
public Object deserialize ( String s, byte [ ] bytes) {
return SerializationUtils. deserialize ( bytes) ;
}
@Override
public void close ( ) { }
}
生产者
package com. baizhi. jsy. serializer;
import org. apache. kafka. clients. producer. KafkaProducer;
import org. apache. kafka. clients. producer. ProducerConfig;
import org. apache. kafka. clients. producer. ProducerRecord;
import org. apache. kafka. common. serialization. StringSerializer;
import java. text. DecimalFormat;
import java. util. Date;
import java. util. Properties;
public class ProductKafkaUser {
public static void main ( String[ ] args) {
Properties properties = new Properties ( ) ;
properties. put ( ProducerConfig. BOOTSTRAP_SERVERS_CONFIG, "Centos:9092" ) ;
properties. put ( ProducerConfig. KEY_SERIALIZER_CLASS_CONFIG, StringSerializer. class . getName ( ) ) ;
properties. put ( ProducerConfig. VALUE_SERIALIZER_CLASS_CONFIG, "com.baizhi.jsy.serializer.Userserializer" ) ;
properties. put ( ProducerConfig. BATCH_SIZE_CONFIG, 1024 * 1024 ) ;
properties. put ( ProducerConfig. LINGER_MS_CONFIG, 500 ) ;
KafkaProducer< String, User> kafkaProducer = new KafkaProducer < String, User> ( properties) ;
for ( int i= 0 ; i< 10 ; i++ ) {
DecimalFormat decimalFormat = new DecimalFormat ( "00" ) ;
String format = decimalFormat. format ( i) ;
ProducerRecord< String, User> record = new ProducerRecord < String, User> ( "topic02" , "key" + format, new User ( format, "User" + format, new Date ( ) ) ) ;
kafkaProducer. send ( record) ;
}
kafkaProducer. flush ( ) ;
kafkaProducer. close ( ) ;
}
}
消费者
Properties props= new Properties ( ) ;
props. put ( ConsumerConfig. BOOTSTRAP_SERVERS_CONFIG, "CentOSA:9092,CentOSB:9092,CentOSC:9092" ) ;
props. put ( ConsumerConfig. KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer. class . getName ( ) ) ;
props. put ( ConsumerConfig. VALUE_DESERIALIZER_CLASS_CONFIG, ObjectDeserializer. class . getName ( ) ) ;
props. put ( ConsumerConfig. GROUP_ID_CONFIG, "group01" ) ;
KafkaConsumer< String, User> consumer= new KafkaConsumer < String, User> ( props) ;
consumer. subscribe ( Pattern. compile ( "^topic.*$" ) ) ;
while ( true ) {
ConsumerRecords< String, User> consumerRecords = consumer. poll ( Duration. ofSeconds ( 1 ) ) ;
Iterator< ConsumerRecord< String, User> > recordIterator = consumerRecords. iterator ( ) ;
while ( recordIterator. hasNext ( ) ) {
ConsumerRecord< String, User> record = recordIterator. next ( ) ;
String key = record. key ( ) ;
User value = record. value ( ) ;
long offset = record. offset ( ) ;
int partition = record. partition ( ) ;
System. out. println ( "key:" + key+ ",value:" + value+ ",partition:" + partition+ ",offset:" + offset) ;
}
}
自定义分区
源代码
package org. apache. kafka. clients. producer. internals;
import java. util. List;
import java. util. Map;
import java. util. concurrent. ConcurrentHashMap;
import java. util. concurrent. ConcurrentMap;
import java. util. concurrent. ThreadLocalRandom;
import java. util. concurrent. atomic. AtomicInteger;
import org. apache. kafka. clients. producer. Partitioner;
import org. apache. kafka. common. Cluster;
import org. apache. kafka. common. PartitionInfo;
import org. apache. kafka. common. utils. Utils;
public class DefaultPartitioner implements Partitioner {
private final ConcurrentMap< String, AtomicInteger> topicCounterMap = new ConcurrentHashMap < > ( ) ;
public void configure ( Map< String, ? > configs) { }
public int partition ( String topic, Object key, byte [ ] keyBytes, Object value, byte [ ] valueBytes, Cluster cluster) {
List< PartitionInfo> partitions = cluster. partitionsForTopic ( topic) ;
int numPartitions = partitions. size ( ) ;
if ( keyBytes == null) {
int nextValue = nextValue ( topic) ;
List< PartitionInfo> availablePartitions = cluster. availablePartitionsForTopic ( topic) ;
if ( availablePartitions. size ( ) > 0 ) {
int part = Utils. toPositive ( nextValue) % availablePartitions. size ( ) ;
return availablePartitions. get ( part) . partition ( ) ;
} else {
return Utils. toPositive ( nextValue) % numPartitions;
}
} else {
return Utils. toPositive ( Utils. murmur2 ( keyBytes) ) % numPartitions;
}
}
private int nextValue ( String topic) {
AtomicInteger counter = topicCounterMap. get ( topic) ;
if ( null == counter) {
counter = new AtomicInteger ( ThreadLocalRandom. current ( ) . nextInt ( ) ) ;
AtomicInteger currentCounter = topicCounterMap. putIfAbsent ( topic, counter) ;
if ( currentCounter != null) {
counter = currentCounter;
}
}
return counter. getAndIncrement ( ) ;
}
public void close ( ) { }
}
自定义分区
package com. baizhi. jsy. Partitions;
import org. apache. kafka. clients. producer. Partitioner;
import org. apache. kafka. common. Cluster;
import org. apache. kafka. common. PartitionInfo;
import org. apache. kafka. common. utils. Utils;
import java. util. List;
import java. util. Map;
import java. util. concurrent. ConcurrentHashMap;
import java. util. concurrent. ConcurrentMap;
import java. util. concurrent. ThreadLocalRandom;
import java. util. concurrent. atomic. AtomicInteger;
public class UserDefinePartitioner implements Partitioner {
private final ConcurrentMap< String, AtomicInteger> topicCounterMap = new ConcurrentHashMap < > ( ) ;
@Override
public int partition ( String topic, Object key, byte [ ] keyBytes, Object value, byte [ ] valueBytes, Cluster cluster) {
List< PartitionInfo> partitionInfos = cluster. partitionsForTopic ( topic) ;
int numPartitions = partitionInfos. size ( ) ;
System. out. println ( "numPartitions = " + numPartitions) ;
if ( keyBytes== null || keyBytes. length== 0 ) {
int nextValue = nextValue ( topic) ;
Integer lx= Utils. toPositive ( nextValue) % numPartitions;
System. out. println ( "lx = " + lx) ;
return lx;
} else {
Integer p = Utils. toPositive ( Utils. murmur2 ( keyBytes) ) % numPartitions;
System. out. println ( "p = " + p) ;
return p;
}
}
private int nextValue ( String topic) {
AtomicInteger counter = topicCounterMap. get ( topic) ;
if ( null == counter) {
counter = new AtomicInteger ( ThreadLocalRandom. current ( ) . nextInt ( ) ) ;
AtomicInteger currentCounter = topicCounterMap. putIfAbsent ( topic, counter) ;
if ( currentCounter != null) {
counter = currentCounter;
}
}
return counter. getAndIncrement ( ) ;
}
@Override
public void close ( ) {
System. out. println ( "close" ) ;
}
@Override
public void configure ( Map< String, ? > configs) {
System. out. println ( "configure" ) ;
}
}
生产者
package com. baizhi. jsy. Partitions;
import org. apache. kafka. clients. producer. KafkaProducer;
import org. apache. kafka. clients. producer. ProducerConfig;
import org. apache. kafka. clients. producer. ProducerRecord;
import org. apache. kafka. common. serialization. StringSerializer;
import java. text. DecimalFormat;
import java. util. Date;
import java. util. Properties;
public class PartitionProductKafkaUser {
public static void main ( String[ ] args) {
Properties properties = new Properties ( ) ;
properties. put ( ProducerConfig. BOOTSTRAP_SERVERS_CONFIG, "Centos:9092" ) ;
properties. put ( ProducerConfig. KEY_SERIALIZER_CLASS_CONFIG, StringSerializer. class . getName ( ) ) ;
properties. put ( ProducerConfig. VALUE_SERIALIZER_CLASS_CONFIG, "com.baizhi.jsy.serializer.Userserializer" ) ;
properties. put ( ProducerConfig. PARTITIONER_CLASS_CONFIG, "com.baizhi.jsy.Partitions.UserDefinePartitioner" ) ;
properties. put ( ProducerConfig. BATCH_SIZE_CONFIG, 1024 * 1024 ) ;
properties. put ( ProducerConfig. LINGER_MS_CONFIG, 500 ) ;
KafkaProducer< String, User> kafkaProducer = new KafkaProducer < String, User> ( properties) ;
for ( int i= 0 ; i< 10 ; i++ ) {
DecimalFormat decimalFormat = new DecimalFormat ( "00" ) ;
String format = decimalFormat. format ( i) ;
ProducerRecord< String, User> record = new ProducerRecord < String, User> ( "topic02" , new User ( format, "User" + format, new Date ( ) ) ) ;
kafkaProducer. send ( record) ;
}
kafkaProducer. flush ( ) ;
kafkaProducer. close ( ) ;
}
}
消费者
package com. baizhi. jsy. Partitions;
import org. apache. kafka. clients. consumer. ConsumerConfig;
import org. apache. kafka. clients. consumer. ConsumerRecord;
import org. apache. kafka. clients. consumer. ConsumerRecords;
import org. apache. kafka. clients. consumer. KafkaConsumer;
import org. apache. kafka. common. serialization. StringDeserializer;
import java. time. Duration;
import java. util. Arrays;
import java. util. Iterator;
import java. util. Properties;
public class PartitionConsumerKafkaUser {
public static void main ( String[ ] args) {
Properties properties = new Properties ( ) ;
properties. put ( ConsumerConfig. BOOTSTRAP_SERVERS_CONFIG, "Centos:9092" ) ;
properties. put ( ConsumerConfig. KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer. class . getName ( ) ) ;
properties. put ( ConsumerConfig. VALUE_DESERIALIZER_CLASS_CONFIG, "com.baizhi.jsy.serializer.UserDeserializer" ) ;
properties. put ( ConsumerConfig. GROUP_ID_CONFIG, "g2" ) ;
KafkaConsumer< String, User> kafkaConsumer = new KafkaConsumer < > ( properties) ;
kafkaConsumer. subscribe ( Arrays. asList ( "topic02" ) ) ;
try {
while ( true ) {
ConsumerRecords< String, User> consumerRecords = kafkaConsumer. poll ( Duration. ofSeconds ( 1 ) ) ;
if ( ! consumerRecords. isEmpty ( ) ) {
Iterator< ConsumerRecord< String, User> > iterator = consumerRecords. iterator ( ) ;
while ( iterator. hasNext ( ) ) {
ConsumerRecord< String, User> next = iterator. next ( ) ;
String topic = next. topic ( ) ;
String key = next. key ( ) ;
User user = next. value ( ) ;
long offset = next. offset ( ) ;
int partition = next. partition ( ) ;
long timestamp = next. timestamp ( ) ;
System. out. println ( "topic = " + topic+ "\t" + "key = " + key+ "\t" + "User = " + user+ "\t" + "offset = " + offset+ "\t" + "partition = " + partition+ "\t" + "timestamp = " + timestamp) ;
System. out. println ( ) ;
}
}
}
} catch ( Exception e) {
e. printStackTrace ( ) ;
} finally {
kafkaConsumer. close ( ) ;
}
}
}