spark持久化级别和java的持久化

/**
*

持久化操作

*/
package com.aura.liu.Dayof20

import org.apache.log4j.{Level, Logger}
import org.apache.spark.rdd.RDD
import org.apache.spark.storage.StorageLevel
import org.apache.spark.{SparkConf, SparkContext}

/**

• 持久化操作
  */
  object PerSistApp {
  def main(args: Array[String]): Unit = {
  //局部调整spark应用的日志级别
  Logger.getLogger(“org.apache.spark”).setLevel(Level.WARN);
  Logger.getLogger(“org.apache.hadoop”).setLevel(Level.WARN);
  Logger.getLogger(“org.spark_project”).setLevel(Level.WARN);

    val conf=new SparkConf()
    .setAppName(s"${PerSistApp.getClass.getSimpleName}")
    .setMaster("local[*]")
  
    val  sc=new SparkContext(conf)
    //开始时间
    var start=System.currentTimeMillis()
    val listRDD:RDD[String]=sc.textFile("file:///D:/stutus.txt")
    //设置持久化级别
    listRDD.persist(StorageLevel.MEMORY_ONLY)
    var retRDD:RDD[(String,Int)]=listRDD.flatMap(_.split(("\\s+")).map((_,1))).reduceByKey(_+_)
    println("#######count s:"+retRDD.count())
    //结束时间
    var  end=System.currentTimeMillis()
    println("未持久化消耗时间：" +(end-start) +"ms")
    println("======================")
    //开始时间
    start=System.currentTimeMillis()
    retRDD=listRDD.flatMap(_.split("\\s+")).map((_,1)).reduceByKey(_+_)
    println("########counts:"+retRDD.count())
    //结束时间
    end=System.currentTimeMillis()
    println("持久化消耗时间："+(end-start)+"ms")
    sc.stop()
  

}
}

java持久化

package Demo20Test;

import java.io.*;

public class SerilizationOPs {
public static void main(String[] args) throws IOException, ClassNotFoundException {

    String filePath = "data/obj.txt";
            writeOb2File(filePath);
            readObjFromFile(filePath);

}
//读取对象

private static void readObjFromFile(String filepath) throws IOException, ClassNotFoundException {
//文件输入流
InputStream fs=new FileInputStream(filepath);
//对象输入流
ObjectInputStream ois=new ObjectInputStream(fs);
//读取对象
Object obj=ois.readObject();
//判断对象的类型
if (obj instanceof Person[]){
//对象类型的转换
Person[] ps= (Person[]) obj;
//循环遍历输出
for (Person p:ps){
System.out.println§;

}}

}

public static void writeOb2File(String filepath) throws IOException {
    Person p1 = new Person("李娥飞", 20);
    Person p2 = new Person("刘琳峰", 28);
    Person p3 = new Person("朱梅星", 24);

    /**
     * 进行持久化操作将person对象写入到文件中
     * 运用文件流
     * ObjectOutputStream
     */
    //文件输出流
    FileOutputStream fos=new FileOutputStream(filepath);
    //对象输出流
    ObjectOutputStream oos=new ObjectOutputStream(fos);
    //将对象发入到数组中
    Person[] pe={p1,p2,p3};
    //将对象写入到文件中
    oos.writeObject(pe);
    fos.close();

}

} //序列化 实现Serializable将person进行序列化
class Person implements Serializable{
String name;
int age;

    public Person(String name,int age){
        this.name=name;
        this.age=age;
    }

    @Override
    public String toString() {
        return this.name+"-----"+this.age;
    }
}