SpringBoot集成AVRO序列化

SpringBoot集成AVRO序列化

1.背景

上游服务要求下游服务发送的数据要进行avro序列化之后再发送，然后下游服务通过定义数据的schema标签进行序列化后，上游获取schema标签进行反序列化，从而获得原数据。虽然有点恶心，但是下游服务开干！

2.引入依赖

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <parent>
        <artifactId>springboot-common-test</artifactId>
        <groupId>com.gwliuc</groupId>
        <version>1.0-SNAPSHOT</version>
    </parent>
    <modelVersion>4.0.0</modelVersion>

    <artifactId>serilize-avro</artifactId>
    <dependencies>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-web</artifactId>
        </dependency>
        <dependency>
            <groupId>commons-lang</groupId>
            <artifactId>commons-lang</artifactId>
            <version>2.1</version>
        </dependency>
        <dependency>
            <groupId>commons-io</groupId>
            <artifactId>commons-io</artifactId>
            <version>2.5</version>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-test</artifactId>
        </dependency>
        <dependency>
            <groupId>org.apache.avro</groupId>
            <artifactId>avro</artifactId>
            <version>1.8.2</version>
        </dependency>
        <dependency>
            <groupId>org.projectlombok</groupId>
            <artifactId>lombok</artifactId>
        </dependency>
    </dependencies>
    <build>
        <plugins>
            <plugin>
                <groupId>org.apache.avro</groupId>
                <artifactId>avro-maven-plugin</artifactId>
                <version>1.8.2</version>
                <executions>
                    <execution>
                        <phase>generate-sources</phase>
                        <goals>
                            <goal>schema</goal>
                        </goals>
                        <configuration>
                            <!-- 配置schema文件目录 -->
                            <sourceDirectory>${project.basedir}/src/main/resources/avro</sourceDirectory>
                            <!-- 对应自动生成类的生成目录 -->
                            <outputDirectory>${project.basedir}/src/main/java/</outputDirectory>
                        </configuration>
                    </execution>
                </executions>
            </plugin>

        </plugins>
    </build>
</project>

3.定义序列化和反序列化方法

package com.avro.util;

import org.apache.avro.Schema;
import org.apache.avro.io.*;
import org.apache.avro.specific.SpecificDatumReader;
import org.apache.avro.specific.SpecificDatumWriter;
import org.apache.commons.io.IOUtils;

import java.io.ByteArrayOutputStream;

/**
 * avro工具类
 */
public class MyAvroUtil {

    /**
     * 根据不同类型的数据AVRO序列化为字节数组
     * @param data 泛型数据
     * @param schema 模式
     * @param <T> 泛型
     * @return byte[] 字节数组
     */
    public static <T> byte[] serializeData(T data, Schema schema) {
        ByteArrayOutputStream outputStream = null;
        try{
            outputStream = new ByteArrayOutputStream();
            DatumWriter<T> writer = new SpecificDatumWriter<>(schema);
            BinaryEncoder binaryEncoder = EncoderFactory.get().directBinaryEncoder(outputStream, null);
            writer.setSchema(schema);
            writer.write(data, binaryEncoder);
            outputStream.flush();
            //序列化后的字节数组
            return outputStream.toByteArray();
        }catch (Exception e) {
            throw new RuntimeException(e);
        } finally {
            IOUtils.closeQuietly(outputStream);
        }
    }

    /**
     * 根据字节数组反序列化为指定类型
     *
     * @param cls 指定类型
     * @param schema 模式
     * @param byteArray 字节数组
     * @param <T> 泛型
     * @return T 具体类型数据
     */
    public static <T>T deSerializeData(Class<T> cls, Schema schema, byte[] byteArray) {
        try {
            T data = null;
            DatumReader<T> datumReader = new SpecificDatumReader<T>(schema);
            BinaryDecoder binaryDecoder = DecoderFactory.get().binaryDecoder(byteArray, null);
            datumReader.setSchema(schema);
            if (!binaryDecoder.isEnd()) {
                data = datumReader.read(null, binaryDecoder);
            }
            return data;
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }
}

4.创建schema标签

例如：data.avsc放到resource目录，如下

{
  "type": "record",
  "namespace": "com.avro.entity",
  "name": "DataRecord",
  "fields": [
    {
      "type": "int",
      "name": "id",
      "default": 0
    },
    {
      "type": "string",
      "name": "desc",
      "default": ""
    }
  ]
}

这里面的namespace必须写，不然序列化和反序列化报错！！！！

5.项目编译，生成对应schema映射类

6.测试

测试类

import com.avro.AvroApplication;
import com.avro.entity.DataRecord;
import com.avro.util.MyAvroUtil;
import lombok.extern.slf4j.Slf4j;
import org.apache.avro.Schema;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit.jupiter.SpringExtension;

import java.io.IOException;

@ExtendWith(SpringExtension.class)
@SpringBootTest(classes = AvroApplication.class)
@Slf4j
public class AvroTest {
    private static final String schemaPath = "avro/data.avsc";
    @Test
    public void testAvro() throws IOException {
        // 构造数据
        DataRecord dataRecord = DataRecord.newBuilder().setId(12).setDesc("aaa").build();
        log.info("原数据为{}",dataRecord);
        //获取schema
        Schema schema = new Schema.Parser().parse(AvroTest.class.getClassLoader().getResourceAsStream(schemaPath));
        log.info("获取schema>>>>>{}", new Object[]{schema});
        // 序列化
        byte[] bytes = MyAvroUtil.serializeData(dataRecord, schema);
        log.info("avro序列化的字节数组{}", new Object[]{bytes});
        // 反序列化
        DataRecord dataRecord1 = MyAvroUtil.deSerializeData(DataRecord.class, schema, bytes);
        log.info("反序列后的数据为{}", dataRecord1);
    }

}

截图