接上篇Linux系统Kafka集群搭建与简单测试,做简单的测试
新建common工程
- 新建listen-kafka-common工程,主要放置生产端和消费端公用部分代码
- 工程目录截图如下图
- pom.xml文件如下
-
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion>4.0.0</modelVersion> <parent> <groupId>listen-kafka</groupId> <artifactId>listen-kafka-parent</artifactId> <version>0.0.1-SNAPSHOT</version> </parent> <packaging>jar</packaging> <artifactId>listen-kafka-common</artifactId> <dependencies> <dependency> <groupId>com.alibaba</groupId> <artifactId>fastjson</artifactId> <version>1.1.41</version> </dependency> <dependency> <groupId>org.apache.kafka</groupId> <artifactId>kafka_2.10</artifactId> <version>0.10.0.0</version> </dependency> </dependencies> </project>
编写代码
- 实体类User
-
package com.listen.kafka.common.entity; import java.io.Serializable; import com.alibaba.fastjson.JSONObject; public class User implements Serializable { private String name; private int age; private boolean sex; public User() { } public User(String name, int age, boolean sex) { this.name = name; this.age = age; this.sex = sex; } @Override public String toString(){ return JSONObject.toJSONString(this); } public String getName() { return name; } public void setName(String name) { this.name = name; } public int getAge() { return age; } public void setAge(int age) { this.age = age; } public boolean isSex() { return sex; } public void setSex(boolean sex) { this.sex = sex; } }
- 序列化类UserSerializer
-
package com.listen.kafka.common.coder; import java.io.UnsupportedEncodingException; import java.util.Map; import org.apache.kafka.common.errors.SerializationException; import org.apache.kafka.common.serialization.Serializer; import com.listen.kafka.common.entity.User; public class UserSerializer implements Serializer<User> { private String encoding = "UTF8"; @Override public void configure(Map<String, ?> configs, boolean isKey) { String propertyName = isKey ? "key.serializer.encoding" : "value.serializer.encoding"; Object encodingValue = configs.get(propertyName); if (encodingValue == null){ encodingValue = configs.get("serializer.encoding"); } if (encodingValue != null && encodingValue instanceof String){ encoding = (String) encodingValue; } } @Override public byte[] serialize(String topic, User data) { try { if (data == null) return null; else return data.toString().getBytes(encoding); } catch (UnsupportedEncodingException e) { throw new SerializationException("Error when serializing T to byte[] due to unsupported encoding " + encoding); } } @Override public void close() { //nothing to do } }
- 反序列化类UserDeserializer
-
package com.listen.kafka.common.coder; import java.io.UnsupportedEncodingException; import java.util.Map; import org.apache.kafka.common.errors.SerializationException; import org.apache.kafka.common.serialization.Deserializer; import com.alibaba.fastjson.JSONObject; import com.listen.kafka.common.entity.User; public class UserDeserializer implements Deserializer<User> { private String encoding = "UTF8"; @Override public void configure(Map<String, ?> configs, boolean isKey) { String propertyName = isKey ? "key.deserializer.encoding" : "value.deserializer.encoding"; Object encodingValue = configs.get(propertyName); if (encodingValue == null){ encodingValue = configs.get("deserializer.encoding"); } if (encodingValue != null && encodingValue instanceof String){ encoding = (String) encodingValue; } } @Override public User deserialize(String topic, byte[] data) { try { if (data == null){ return null; } else{ String json = new String(data, encoding); User user = JSONObject.parseObject(json, User.class); return user; } } catch (UnsupportedEncodingException e) { throw new SerializationException("Error when deserializing byte[] to User due to unsupported encoding " + encoding); } } @Override public void close() { //nothing to do } }
-
新建生产消息工程listen-kafka-producer
- 工程目录结构如下图
- pom.xml文件如下
-
<?xml version="1.0"?> <project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> <modelVersion>4.0.0</modelVersion> <parent> <groupId>listen-kafka</groupId> <artifactId>listen-kafka-parent</artifactId> <version>0.0.1-SNAPSHOT</version> </parent> <groupId>listen-kafka-producer</groupId> <artifactId>listen-kafka-producer</artifactId> <packaging>war</packaging> <dependencies> <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> <version>3.8.1</version> <scope>test</scope> </dependency> <dependency> <groupId>javax.servlet</groupId> <artifactId>servlet-api</artifactId> <version>2.5</version> <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.kafka</groupId> <artifactId>kafka-clients</artifactId> <version>0.10.0.0</version> </dependency> <dependency> <groupId>listen-kafka</groupId> <artifactId>listen-kafka-common</artifactId> <version>0.0.1-SNAPSHOT</version> </dependency> </dependencies> <build> <finalName>listen-kafka-producer</finalName> </build> </project>
- 生产消息类ListenKafkaProducer
-
package com.listen.kafka.producer; import java.util.Properties; import org.apache.kafka.clients.producer.Callback; import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.clients.producer.RecordMetadata; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.listen.kafka.common.entity.User; public class ListenKafkaProducer{ private static Logger log = LoggerFactory.getLogger(ListenKafkaProducer.class); public static void main(String[] args) throws InterruptedException{ KafkaProducer<String, User> producer = initProducer(); int i = 0; User user = null; while(true){ user = new User("t-meng" + i, i, i%2 == 0 ? true : false); ProducerRecord<String, User> record = new ProducerRecord<String, User>("t-meng", String.valueOf(i), user); producer.send(record, new Callback() { @Override public void onCompletion(RecordMetadata metadata, Exception exception) { if (exception != null) exception.printStackTrace(); System.out.println("message send to partition " + metadata.partition() + ", offset: " + metadata.offset()); } }); i++; Thread.sleep(1000); } } private static KafkaProducer<String, User> initProducer(){ Properties props = new Properties(); props.put("bootstrap.servers", "192.168.75.141:9092,192.168.75.141:9093,192.168.75.141:9094"); props.put("acks", "all"); props.put("retries", 0); props.put("batch.size", 16384); props.put("linger.ms", 1); props.put("buffer.memory", 33554432); // props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer"); // props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer"); props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer"); props.put("value.serializer", "com.listen.kafka.common.coder.UserSerializer"); props.put("auto.create.topics.enable", true); return new KafkaProducer<String, User>(props); } }
-
新建消费消息工程listen-kafka-consumer
- 工程目录结构如下图
- pom.xml文件,如下
-
<?xml version="1.0"?> <project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> <modelVersion>4.0.0</modelVersion> <parent> <groupId>listen-kafka</groupId> <artifactId>listen-kafka-parent</artifactId> <version>0.0.1-SNAPSHOT</version> </parent> <groupId>listen-kafka-consumer</groupId> <artifactId>listen-kafka-consumer</artifactId> <packaging>war</packaging> <dependencies> <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> <version>3.8.1</version> <scope>test</scope> </dependency> <dependency> <groupId>javax.servlet</groupId> <artifactId>servlet-api</artifactId> <version>2.5</version> <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.kafka</groupId> <artifactId>kafka-clients</artifactId> <version>0.10.0.0</version> </dependency> <dependency> <groupId>org.apache.kafka</groupId> <artifactId>kafka_2.10</artifactId> <version>0.10.0.0</version> </dependency> <dependency> <groupId>listen-kafka</groupId> <artifactId>listen-kafka-common</artifactId> <version>0.0.1-SNAPSHOT</version> </dependency> </dependencies> <build> <finalName>listen-kafka-consumer</finalName> </build> </project>
- 消费消息类ListenKafkaConsumer
-
package com.listen.kafka.consumer; import java.util.Arrays; import java.util.Properties; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.KafkaConsumer; import com.listen.kafka.common.entity.User; public class ListenKafkaConsumer { public static void main(String[] args){ KafkaConsumer<String, User> consumer = initConsumer(); consumer.subscribe(Arrays.asList("t-meng")); while(true){ ConsumerRecords<String, User> records = consumer.poll(100); for(ConsumerRecord<String, User> record : records){ System.out.println("fetched from partition " + record.partition() + ", offset: " + record.offset() + ", message: " + record.value().toString()); } } } private static KafkaConsumer<String, User> initConsumer(){ Properties props = new Properties(); props.put("bootstrap.servers", "192.168.75.141:9092,192.168.75.141:9093,192.168.75.141:9094"); props.put("group.id", "group1"); /** * latest 从消费到的最新的offset位置开始消费 * earliest 从最初的offset位置开始消费,如果是新的topic话 * none 从当前produce生产的最新的offset开始消费,默认 */ props.put("auto.offset.reset", "earliest"); props.put("enable.auto.commit", "true"); props.put("auto.commit.interval.ms", "1000"); props.put("session.timeout.ms", "30000"); // props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); // props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); props.put("value.deserializer", "com.listen.kafka.common.coder.UserDeserializer"); return new KafkaConsumer<String, User>(props); } }
-
编译启动
- 编译3个工程,分别右键Run As --> Java Application 两个文件:ListenKafkaConsumer和ListenKafkaProducer
- 查看控制台打印的日志信息,可以看到消息正常的生产和消费,简单模拟消息的生产和消费。
- 生产的日志
- 消费的日志
- 码云源码地址:
- https://git.oschina.net/git.listen/listen-kafka-parent
- https://git.oschina.net/git.listen/listen-kafka-common
- https://git.oschina.net/git.listen/listen-kafka-producer
- https://git.oschina.net/git.listen/listen-kafka-consumer
Over!