Spark-Avro学习9之SCALA环境下Avro使用（不生成code方式）

本文链接：https://blog.csdn.net/xubo245/article/details/51296717

本文介绍使用Apache Avro进行数据序列化和反序列化的具体实现过程，包括创建记录、写入Avro文件及从Avro文件读取记录。通过两个示例展示如何操作Avro Schema定义的数据。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

1.与上篇类似，只是代码改下：

package org.apache.spark.avro.learning

import java.io.File
import java.text.SimpleDateFormat
import java.util.Date
import org.apache.avro.Schema
import org.apache.avro.file.DataFileReader
import org.apache.avro.file.DataFileWriter
import org.apache.avro.generic.GenericData
import org.apache.avro.generic.GenericDatumReader
import org.apache.avro.generic.GenericDatumWriter
import org.apache.avro.generic.GenericRecord

object AvroNoCode {

  def main(args: Array[String]): Unit = {
    val schema = new Schema.Parser().parse(new File(
      "file/data/avro/input/user.avsc"));
    val user1 = new GenericData.Record(schema);
    println(user1);
    user1.put("name", "Alyssa");
    user1.put("favorite_number", 256);

    val user2 = new GenericData.Record(schema);
    user2.put("name", "Ben");
    user2.put("favorite_number", 7);
    user2.put("favorite_color", "red");

    println("create user:");
    println(user1);
    println(user2);

    // Serialize user1 and user2 to disk
    val iString = new SimpleDateFormat("yyyyMMddHHmmssSSS")
      .format(new Date());
    val file = new File("file/data/avro/output/users" + iString + ".avro");
    val datumWriter = new GenericDatumWriter[GenericRecord] (
      schema);
    val dataFileWriter = new DataFileWriter[GenericRecord] (
      datumWriter);
    dataFileWriter.create(schema, file);
    dataFileWriter.append(user1);
    dataFileWriter.append(user2);
    dataFileWriter.close();

    // Deserialize users from disk
    val datumReader = new GenericDatumReader[GenericRecord] (
      schema);
   val dataFileReader = new DataFileReader[GenericRecord] (
      file, datumReader);
;
    var  user = null:GenericRecord;
    while (dataFileReader.hasNext()) {
      // Reuse user object by passing it to next(). This saves us from
      // allocating and garbage collecting many objects for files with
      // many items.
      user = dataFileReader.next(user);
      println(user);
    }

    println("end");
  }

}

2.运行结果：

{"name": null, "favorite_number": null, "favorite_color": null}
create user:
{"name": "Alyssa", "favorite_number": 256, "favorite_color": null}
{"name": "Ben", "favorite_number": 7, "favorite_color": "red"}
{"name": "Alyssa", "favorite_number": 256, "favorite_color": null}
{"name": "Ben", "favorite_number": 7, "favorite_color": "red"}
end

文件：

Objavro.schema�{"type":"record","name":"User","namespace":"example.avro","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}