Spark序列化

最新推荐文章于 2022-03-30 10:17:47 发布

huo_火力全开

最新推荐文章于 2022-03-30 10:17:47 发布

阅读量259

点赞数 1

分类专栏： Spark 文章标签： Spark

本文链接：https://blog.csdn.net/huonan_123/article/details/87708490

版权

Spark 专栏收录该内容

14 篇文章 0 订阅

订阅专栏

默认没有序列化(StorageLevel.MEMORY_ONLY)

def main(args: Array[String]) {

    val sparkConf = new SparkConf()
      .setMaster("local[2]")
      .setAppName("SequenceFileApp")
     // .set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
    // .registerKryoClasses(Array(classOf[Student]))

    val sc = new SparkContext(sparkConf)
    val students = ListBuffer[Student]()

    for (i <- 0 to 1000000) {
      students.append(Student(i, "student" + i, 39))
    }

    val studentRDD = sc.parallelize(students)
    studentRDD.persist(StorageLevel.MEMORY_ONLY)
    studentRDD.count()

    Thread.sleep(1000 * 20)
    sc.stop()
  }

如下图 95.4M
使用默认Java序列化

 def main(args: Array[String]) {

    val sparkConf = new SparkConf()
      .setMaster("local[2]")
      .setAppName("SequenceFileApp")
    //  .set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
    // .registerKryoClasses(Array(classOf[Student]))

    val sc = new SparkContext(sparkConf)
    val students = ListBuffer[Student]()

    for (i <- 0 to 1000000) {
      students.append(Student(i, "student" + i, 39))
    }

    val studentRDD = sc.parallelize(students)
    studentRDD.persist(StorageLevel.MEMORY_ONLY_SER)
    studentRDD.count()

    Thread.sleep(1000 * 20)
    sc.stop()
  }

如下图29.4M
使用Kryo的序列化(但是没有注册)

 def main(args: Array[String]) {

    val sparkConf = new SparkConf()
      .setMaster("local[2]")
      .setAppName("SequenceFileApp")
      .set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
    // .registerKryoClasses(Array(classOf[Student]))

    val sc = new SparkContext(sparkConf)
    val students = ListBuffer[Student]()

    for (i <- 0 to 1000000) {
      students.append(Student(i, "student" + i, 39))
    }

    val studentRDD = sc.parallelize(students)
    studentRDD.persist(StorageLevel.MEMORY_ONLY_SER)
    studentRDD.count()

    Thread.sleep(1000 * 20)
    sc.stop()
  }

如下图 61.9M
使用Kryo注册类之后的序列化

 def main(args: Array[String]) {

    val sparkConf = new SparkConf()
      .setMaster("local[2]")
      .setAppName("SequenceFileApp")
      .set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
    .registerKryoClasses(Array(classOf[Student]))

    val sc = new SparkContext(sparkConf)
    val students = ListBuffer[Student]()

    for (i <- 0 to 1000000) {
      students.append(Student(i, "student" + i, 39))
    }

    val studentRDD = sc.parallelize(students)
    studentRDD.persist(StorageLevel.MEMORY_ONLY_SER)
    studentRDD.count()

    Thread.sleep(1000 * 20)
    sc.stop()
  }