一、Rdd转化算子
1、【union】并集;
2、【intersection】交集;
3、【distinct】去重;
4、【join】只对相同的key合并操作;
二、实例
package com.cn.rddOperator
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}
object Transformation02 {
def main(args: Array[String]): Unit = {
val conf = new SparkConf().setAppName("transformation02").setMaster("local[1]")
val sc = new SparkContext(conf)
sc.setLogLevel("WARN")
val rdd01: RDD[Int] = sc.parallelize(List(1,2,3,4))
val rdd02: RDD[Int] = sc.parallelize(List(4,1,5,6))
/**
* 并集:union
*/
val rdd03: RDD[Int] =rdd01 union rdd02
println(rdd03.collect().toBuffer)//ArrayBuffer(1, 2, 3, 4, 4, 1, 5, 6)
/**
* 交集:intersection
*/
val rdd04: RDD[Int] = rdd01.intersection(rdd02)
println(rdd04.collect().toBuffer)//ArrayBuffer(4, 1)
/**
* 去重:distinct
*/
println(rdd03.distinct.collect().toBuffer)//ArrayBuffer(4, 6, 2, 1, 3, 5)
/**
* join:只对相同的key合并操作
*/
val rdd05=sc.parallelize(List(("tom",1),("jack",3),("dack",5)))
val rdd06=sc.parallelize(List(("jack",3),("tom",7),("pig",2)))
val rdd07: RDD[(String, (Int, Int))] =rdd05 join rdd06
println(rdd07.collect().toBuffer)//ArrayBuffer((tom,(1,7)), (jack,(3,3)))
}
}