spark java_Spark-Java-算子

package scala.spark.Day3;

import org.apache.spark.SparkConf;

import org.apache.spark.api.java.JavaRDD;

import org.apache.spark.api.java.JavaSparkContext;

import org.apache.spark.api.java.function.Function;

import org.apache.spark.api.java.function.VoidFunction;

import java.util.Arrays;

import java.util.List;

/**

* Created by Administrator on 2019/10/16.

*/

public class JavaRDDTest {

public static void main(String[] args) {

System.setProperty("hadoop.home.dir", "E:\\hadoop-2.6.0-cdh5.15.0\\hadoop-2.6.0-cdh5.15.0");

//JavaRDD 标准RDD

//JavaPairRDD PairRDD

//JavaDoubleRDD DoubleRDD

//java没有scala隐式转化，生成RDD时候，必须指明是哪种RDD

//实例化驱动器

SparkConf sparkConf = new SparkConf();

sparkConf.setMaster("local");

sparkConf.setAppName("Java RDD");

JavaSparkContext javaSparkContext = new JavaSparkContext(sparkConf);

//javaSparkContext.parallelize() 标准RDD

//javaSparkContext.parallelizePairs() PairRDD

//javaSparkContext.parallelizeDoubles() 数值类型RDD

List list = Arrays.asList(1, 2, 3, 4, 5);

JavaRDD sourceRDD = javaSparkContext.parallelize(list);

//map 算子

// 匿名子类

/*

public interface Function extends Serializable {

R call(T1 v1) throws Exception;

}

T1 RDD中成员的类型

v1 返回的类型

*/

JavaRDD mapRDD = sourceRDD.map(

new Function() {

public Integer call(Integer v1) throws Exception {

return v1 * v1;

}

}

);

/*

public interface VoidFunction extends Serializable {

void call(T t) throws Exception;

}

*/

mapRDD.foreach(

new VoidFunction() {

public void call(Integer integer) throws Exception {

System.out.println(integer);

}

}

);

javaSparkContext.close();

}

}