【RDD Transformation】union、intersection、distinct、join

一、Rdd转化算子

1、【union】并集；

2、【intersection】交集；

3、【distinct】去重；

4、【join】只对相同的key合并操作；

二、实例

package com.cn.rddOperator

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

object Transformation02 {
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setAppName("transformation02").setMaster("local[1]")
    val sc = new SparkContext(conf)
    sc.setLogLevel("WARN")

    val rdd01: RDD[Int] = sc.parallelize(List(1,2,3,4))
    val rdd02: RDD[Int] = sc.parallelize(List(4,1,5,6))
    /**
      * 并集：union
      */
    val rdd03: RDD[Int] =rdd01 union rdd02
    println(rdd03.collect().toBuffer)//ArrayBuffer(1, 2, 3, 4, 4, 1, 5, 6)
    /**
      * 交集：intersection
      */
    val rdd04: RDD[Int] = rdd01.intersection(rdd02)
    println(rdd04.collect().toBuffer)//ArrayBuffer(4, 1)

    /**
      * 去重：distinct
      */
    println(rdd03.distinct.collect().toBuffer)//ArrayBuffer(4, 6, 2, 1, 3, 5)

    /**
      * join:只对相同的key合并操作
      */
    val rdd05=sc.parallelize(List(("tom",1),("jack",3),("dack",5)))
    val rdd06=sc.parallelize(List(("jack",3),("tom",7),("pig",2)))
    val rdd07: RDD[(String, (Int, Int))] =rdd05 join rdd06
    println(rdd07.collect().toBuffer)//ArrayBuffer((tom,(1,7)), (jack,(3,3)))
  }
}