java调用avro_Kafka 中使用 Avro 序列化框架(一)：使用传统的 avro API 自定义序列化类和反序列化类...

关于 avro 的 maven 工程的搭建以及 avro 的入门知识，可以参考： Apache Avro 入门

1. 定义 schema 文件，并编译 maven 工程生成实体类

schema 文件名称为：stock.avsc，内容如下：

{

"namespace": "com.bonc.rdpe.kafka110.beans",

"type": "record",

"name": "Stock",

"fields": [

{"name": "stockCode", "type": "string"},

{"name": "stockName", "type": "string"},

{"name": "tradeTime", "type": "long"},

{"name": "preClosePrice", "type": "float"},

{"name": "openPrice", "type": "float"},

{"name": "currentPrice", "type": "float"},

{"name": "highPrice", "type": "float"},

{"name": "lowPrice", "type": "float"}

]

}

编译 maven 工程生成实体类：

2. 自定义序列化类和反序列化类

(1) 序列化类

package com.bonc.rdpe.kafka110.serializer;

import java.io.ByteArrayOutputStream;

import java.io.IOException;

import java.util.Map;

import org.apache.avro.io.BinaryEncoder;

import org.apache.avro.io.DatumWriter;

import org.apache.avro.io.EncoderFactory;

import org.apache.avro.specific.SpecificDatumWriter;

import org.apache.kafka.common.errors.SerializationException;

import org.apache.kafka.common.serialization.Serializer;

import com.bonc.rdpe.kafka110.beans.Stock;

/**

* @Title AvroSerializer.java

* @Description 使用传统的 Avro API 自定义序列化类

* @Author YangYunhe

* @Date 2018-06-21 16:40:35

*/

public class AvroSerializer implements Serializer {

@Override

public void close() {}

@Override

public void configure(Map arg0, boolean arg1) {}

@Override

public byte[] serialize(String topic, Stock data) {

if(data == null) {

return null;

}

DatumWriter writer = new SpecificDatumWriter<>(data.getSchema());

ByteArrayOutputStream out = new ByteArrayOutputStream();

BinaryEncoder encoder = EncoderFactory.get().directBinaryEncoder(out, null);

try {

writer.write(data, encoder);

}catch (IOException e) {

throw new SerializationException(e.getMessage());

}

return out.toByteArray();

}

}

(2) 反序列化类

package com.bonc.rdpe.kafka110.deserializer;

import java.io.ByteArrayInputStream;

import java.io.IOException;

import java.util.Map;

import org.apache.avro.io.BinaryDecoder;

import org.apache.avro.io.DatumReader;

import org.apache.avro.io.DecoderFactory;

import org.apache.avro.specific.SpecificDatumReader;

import org.apache.kafka.common.serialization.Deserializer;

import com.bonc.rdpe.kafka110.beans.Stock;

/**

* @Title AvroDeserializer.java

* @Description 使用传统的 Avro API 自定义反序列类

* @Author YangYunhe

* @Date 2018-06-21 17:19:40

*/

public class AvroDeserializer implements Deserializer {

@Override

public void close() {}

@Override

public void configure(Map arg0, boolean arg1) {}

@Override

public Stock deserialize(String topic, byte[] data) {

if(data == null) {

return null;

}

Stock stock = new Stock();

ByteArrayInputStream in = new ByteArrayInputStream(data);

DatumReader userDatumReader = new SpecificDatumReader<>(stock.getSchema());

BinaryDecoder decoder = DecoderFactory.get().directBinaryDecoder(in, null);

try {

stock = userDatumReader.read(null, decoder);

} catch (IOException e) {

e.printStackTrace();

}

return stock;

}

}

3. KafkaProducer使用自定义的序列化类发送消息

package com.bonc.rdpe.kafka110.producer;

import java.util.Properties;

import org.apache.kafka.clients.producer.KafkaProducer;

import org.apache.kafka.clients.producer.Producer;

import org.apache.kafka.clients.producer.ProducerRecord;

import org.apache.kafka.clients.producer.RecordMetadata;

import com.bonc.rdpe.kafka110.beans.Stock;

/**

* @Title TraditionalAvroProducer.java

* @Description Kafka Producer 发送avro序列化后的Stock对象

* @Author YangYunhe

* @Date 2018-06-21 17:41:59

*/

public class TraditionalAvroProducer {

public static void main(String[] args) throws Exception {

Stock[] stocks = new Stock[100];

for(int i = 0; i < 100; i++) {

stocks[i] = new Stock();

stocks[i].setStockCode(String.valueOf(i));

stocks[i].setStockName("stock" + i);

stocks[i].setTradeTime(System.currentTimeMillis());

stocks[i].setPreClosePrice(100.0F);

stocks[i].setOpenPrice(88.8F);

stocks[i].setCurrentPrice(120.5F);

stocks[i].setHighPrice(300.0F);

stocks[i].setLowPrice(12.4F);

}

Properties props = new Properties();

props.put("bootstrap.servers", "192.168.42.89:9092,192.168.42.89:9093,192.168.42.89:9094");

props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");

// 设置序列化类为自定义的 avro 序列化类

props.put("value.serializer", "com.bonc.rdpe.kafka110.serializer.AvroSerializer");

Producer producer = new KafkaProducer<>(props);

for(Stock stock : stocks) {

ProducerRecord record = new ProducerRecord<>("dev3-yangyunhe-topic001", stock);

RecordMetadata metadata = producer.send(record).get();

StringBuilder sb = new StringBuilder();

sb.append("stock: ").append(stock.toString()).append(" has been sent successfully!").append("\n")

.append("send to partition ").append(metadata.partition())

.append(", offset = ").append(metadata.offset());

System.out.println(sb.toString());

Thread.sleep(100);

}

producer.close();

}

}

4. KafkaConsumer使用自定义的反序列化类接收消息

package com.bonc.rdpe.kafka110.consumer;

import java.util.Collections;

import java.util.Properties;

import org.apache.kafka.clients.consumer.ConsumerRecord;

import org.apache.kafka.clients.consumer.ConsumerRecords;

import org.apache.kafka.clients.consumer.KafkaConsumer;

import com.bonc.rdpe.kafka110.beans.Stock;

/**

* @Title TraditionalAvroConsumer.java

* @Description Kafka Consumer 解析avro序列化后的Stock对象

* @Author YangYunhe

* @Date 2018-06-21 17:43:03

*/

public class TraditionalAvroConsumer {

public static void main(String[] args) {

Properties props = new Properties();

props.put("bootstrap.servers", "192.168.42.89:9092,192.168.42.89:9093,192.168.42.89:9094");

props.put("group.id", "dev3-yangyunhe-group001");

props.put("key.deserializer","org.apache.kafka.common.serialization.StringDeserializer");

// 设置反序列化类为自定义的avro反序列化类

props.put("value.deserializer","com.bonc.rdpe.kafka110.deserializer.AvroDeserializer");

KafkaConsumer consumer = new KafkaConsumer<>(props);

consumer.subscribe(Collections.singletonList("dev3-yangyunhe-topic001"));

try {

while(true) {

ConsumerRecords records = consumer.poll(100);

for(ConsumerRecord record : records) {

Stock stock = record.value();

System.out.println(stock.toString());

}

}

}finally {

consumer.close();

}

}

}

5. 测试结果

运行生产者代码后控制台输出：

stock: {"stockCode": "0", "stockName": "stock0", "tradeTime": 1529631848353, "preClosePrice": 100.0, "openPrice": 88.8, "currentPrice": 120.5, "highPrice": 300.0, "lowPrice": 12.4} has been sent successfully!

send to partition 0, offset = 552

stock: {"stockCode": "1", "stockName": "stock1", "tradeTime": 1529631848353, "preClosePrice": 100.0, "openPrice": 88.8, "currentPrice": 120.5, "highPrice": 300.0, "lowPrice": 12.4} has been sent successfully!

send to partition 2, offset = 551

stock: {"stockCode": "2", "stockName": "stock2", "tradeTime": 1529631848353, "preClosePrice": 100.0, "openPrice": 88.8, "currentPrice": 120.5, "highPrice": 300.0, "lowPrice": 12.4} has been sent successfully!

send to partition 1, offset = 551

stock: {"stockCode": "3", "stockName": "stock3", "tradeTime": 1529631848353, "preClosePrice": 100.0, "openPrice": 88.8, "currentPrice": 120.5, "highPrice": 300.0, "lowPrice": 12.4} has been sent successfully!

send to partition 0, offset = 553

stock: {"stockCode": "4", "stockName": "stock4", "tradeTime": 1529631848353, "preClosePrice": 100.0, "openPrice": 88.8, "currentPrice": 120.5, "highPrice": 300.0, "lowPrice": 12.4} has been sent successfully!

send to partition 2, offset = 552

......

运行消费者代码后控制台输出：

{"stockCode": "0", "stockName": "stock0", "tradeTime": 1529631848353, "preClosePrice": 100.0, "openPrice": 88.8, "currentPrice": 120.5, "highPrice": 300.0, "lowPrice": 12.4}

{"stockCode": "1", "stockName": "stock1", "tradeTime": 1529631848353, "preClosePrice": 100.0, "openPrice": 88.8, "currentPrice": 120.5, "highPrice": 300.0, "lowPrice": 12.4}

{"stockCode": "2", "stockName": "stock2", "tradeTime": 1529631848353, "preClosePrice": 100.0, "openPrice": 88.8, "currentPrice": 120.5, "highPrice": 300.0, "lowPrice": 12.4}

{"stockCode": "3", "stockName": "stock3", "tradeTime": 1529631848353, "preClosePrice": 100.0, "openPrice": 88.8, "currentPrice": 120.5, "highPrice": 300.0, "lowPrice": 12.4}

{"stockCode": "4", "stockName": "stock4", "tradeTime": 1529631848353, "preClosePrice": 100.0, "openPrice": 88.8, "currentPrice": 120.5, "highPrice": 300.0, "lowPrice": 12.4}

......