Spark sql avro

import org.apache.avro.Schema

import org.apache.avro.generic.GenericData

import org.apache.avro.generic.GenericRecord

import org.apache.avro.generic.GenericRecordBuilder

import org.apache.spark.sql.functions.{col, struct}

import org.apache.spark.sql.avro.functions._

 

// 定义 Avro schema

val avroSchema = new Schema.Parser().parse("{\"type\":\"record\",\"name\":\"myrecord\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"age\",\"type\":\"int\"}]}")

 

// 创建一个 DataFrame

val df = Seq(("Alice", 25), ("Bob", 30)).toDF("name", "age")

 

// 将 DataFrame 中的数据类型转换为 Avro schema 中的数据类型

val avroData = df.select(struct(col("name"), col("age")).as("data")).select(to_avro(col("data"), avroSchema).as("avroData"))

 

// 将 Avro 数据序列化为字节流

val avroBytes = avroData.select("avroData").head.getAs[Array[Byte]]("avroData")

 

// 将字节流反序列化为 Avro record

val avroRecord = new GenericData.Record(avroSchema)

val reader = new org.apache.avro.io.DecoderFactory().binaryDecoder(avroBytes, null)

new org.apache.avro.generic.GenericDatumReader[GenericRecord](avroSchema).read(avroRecord, reader)

 

// 输出 Avro record

println(avroRecord)