- 默认没有序列化(StorageLevel.MEMORY_ONLY)
def main(args: Array[String]) {
val sparkConf = new SparkConf()
.setMaster("local[2]")
.setAppName("SequenceFileApp")
// .set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
// .registerKryoClasses(Array(classOf[Student]))
val sc = new SparkContext(sparkConf)
val students = ListBuffer[Student]()
for (i <- 0 to 1000000) {
students.append(Student(i, "student" + i, 39))
}
val studentRDD = sc.parallelize(students)
studentRDD.persist(StorageLevel.MEMORY_ONLY)
studentRDD.count()
Thread.sleep(1000 * 20)
sc.stop()
}
- 如下图 95.4M
- 使用默认Java序列化
def main(args: Array[String]) {
val sparkConf = new SparkConf()
.setMaster("local[2]")
.setAppName("SequenceFileApp")
// .set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
// .registerKryoClasses(Array(classOf[Student]))
val sc = new SparkContext(sparkConf)
val students = ListBuffer[Student]()
for (i <- 0 to 1000000) {
students.append(Student(i, "student" + i, 39))
}
val studentRDD = sc.parallelize(students)
studentRDD.persist(StorageLevel.MEMORY_ONLY_SER)
studentRDD.count()
Thread.sleep(1000 * 20)
sc.stop()
}
-
如下图29.4M
-
使用Kryo的序列化(但是没有注册)
def main(args: Array[String]) {
val sparkConf = new SparkConf()
.setMaster("local[2]")
.setAppName("SequenceFileApp")
.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
// .registerKryoClasses(Array(classOf[Student]))
val sc = new SparkContext(sparkConf)
val students = ListBuffer[Student]()
for (i <- 0 to 1000000) {
students.append(Student(i, "student" + i, 39))
}
val studentRDD = sc.parallelize(students)
studentRDD.persist(StorageLevel.MEMORY_ONLY_SER)
studentRDD.count()
Thread.sleep(1000 * 20)
sc.stop()
}
- 如下图 61.9M
- 使用Kryo注册类之后的序列化
def main(args: Array[String]) {
val sparkConf = new SparkConf()
.setMaster("local[2]")
.setAppName("SequenceFileApp")
.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
.registerKryoClasses(Array(classOf[Student]))
val sc = new SparkContext(sparkConf)
val students = ListBuffer[Student]()
for (i <- 0 to 1000000) {
students.append(Student(i, "student" + i, 39))
}
val studentRDD = sc.parallelize(students)
studentRDD.persist(StorageLevel.MEMORY_ONLY_SER)
studentRDD.count()
Thread.sleep(1000 * 20)
sc.stop()
}
- 如下图 19.0M
结论:使用Kryo序列化后,文件更小了