package day11;
import javax.sound.sampled.Line;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.storage.StorageLevel;
public class PersistTest {
public static void main(String[] args) {
SparkConf conf=new SparkConf();
conf.setAppName("test");
conf.setMaster("local");
JavaSparkContext sc=new JavaSparkContext(conf);
sc.setCheckpointDir("./checkpointDIR");
JavaRDD<String> Lines=sc.textFile("./words");
Lines.checkpoint();
Lines.count();
long startTime1=System.currentTimeMillis();
JavaRDD<String> line=sc.textFile("./words");
line=line.cache();
line=line.persist(StorageLevel.MEMORY_ONLY());
line.collect();
long count1=line.count();
long endTime1=System.currentTimeMillis();
System.out.println("count1="+count1+",time="+(endTime1-startTime1)+"ms");
long startTime2=System.currentTimeMillis();
long count2=Lines.count();
long endTime2=System.currentTimeMillis();
System.out.println("count2="+count2+",time="+(endTime2-startTime2)+"ms");
sc.stop();
}
}
Spark部分:PersistTest(包含RDD读取数据和写数据到内存中,计算执行时间,【Java版本】)(纯代码)
最新推荐文章于 2023-03-13 15:28:56 发布