import org.apache.avro.Schema
import org.apache.avro.generic.GenericData
import org.apache.avro.generic.GenericRecord
import org.apache.avro.generic.GenericRecordBuilder
import org.apache.spark.sql.functions.{col, struct}
import org.apache.spark.sql.avro.functions._
// 定义 Avro schema
val avroSchema = new Schema.Parser().parse("{\"type\":\"record\",\"name\":\"myrecord\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"age\",\"type\":\"int\"}]}")
// 创建一个 DataFrame
val df = Seq(("Alice", 25), ("Bob", 30)).toDF("name", "age")
// 将 DataFrame 中的数据类型转换为 Avro schema 中的数据类型
val avroData = df.select(struct(col("name"), col("age")).as("data")).select(to_avro(col("data"), avroSchema).as("avroData"))
// 将 Avro 数据序列化为字节流
val avroBytes = avroData.select("avroData").head.getAs[Array[Byte]]("avroData")
// 将字节流反序列化为 Avro record
val avroRecord = new GenericData.Record(avroSchema)
val reader = new org.apache.avro.io.DecoderFactory().binaryDecoder(avroBytes, null)
new org.apache.avro.generic.GenericDatumReader[GenericRecord](avroSchema).read(avroRecord, reader)
// 输出 Avro record
println(avroRecord)