现在是2019年11月14日22点11分,刚看完尚硅谷老师spark的第二天视频,对于老师讲RDD中的算子特别感兴趣,于是写在这里记录一下
目录
2.改变数据结构,相当于数据的批处理(mapPartitions)
3.分片的索引值,找出数据及其所在分区(mapPartitionsWithIndex)
一、value类型
1、转换数据结构,形成新的RDD(map)
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}
object SparkOperator {
def main(args: Array[String]): Unit = {
//获取sparkkconf
val conf: SparkConf = new SparkConf().setAppName("SparkOperator").setMaster("local[*]")
//获取sparkcontext
val sc = new SparkContext(conf)
//创建一个rdd
val listRDD: RDD[Int] = sc.makeRDD(1 to 10)
//转换数据结构。形成新的RDD
val mapRDD: RDD[Int] = listRDD.map(_ * 2)
//打印
mapRDD.collect().foreach(println)
}
}
2.改变数据结构,相当于数据的批处理(mapPartitions)
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}
object SparkOperator01 {
def main(args: Array[String]): Unit = {
//获取sparkkconf
val conf: SparkConf = new SparkConf().setAppName("SparkOperator").setMaster("local[*]")
//获取sparkcontext
val sc = new SparkContext(conf)
//创建一个rdd
val listRDD: RDD[Int] = sc.makeRDD(1 to 10)
//改变数据结构,相当于数据的批处理
/*val mapPartitionsRDD: RDD[Int] = listRDD.mapPartitions(datas => {
datas.map(_ * 2)
})*/
val mapPartitionsRDD: RDD[Int] = listRDD.mapPartitions(_.map(_ * 2))
//打印
mapPartitionsRDD.collect().foreach(println)
}
}
3.分片的索引值,找出数据及其所在分区(mapPartitionsWithIndex)
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}
object SparkOperator02 {
def main(args: Array[String]): Unit = {
//获取sparkkconf
val conf: SparkConf = new SparkConf().setAppName("SparkOperator").setMaster("local[*]")
//获取sparkcontext
val sc = new SparkContext(conf)
//创建一个rdd
val listRDD: RDD[Int] = sc.makeRDD(1 to 10,2)
//打印出每个数和所在的分区
val indexRDD: RDD[(Int, String)] = listRDD.mapPartitionsWithIndex {
case (num, datas) => {
datas.map((_,"分区号:"+num))
}
}
indexRDD.collect().foreach(println)
}
}
4.将每一个分区形成一个新的数组(glom)
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}
object SparkOperator03 {
def main(args: Array[String]): Unit = {
//获取sparkkconf
val conf: SparkConf = new SparkConf().setAppName("SparkOperator").setMaster("local[*]")
//获取sparkcontext
val sc = new SparkContext(conf)
//创建一个glomrdd
val listRDD: RDD[Int] = sc.makeRDD(List(1,2,3,4),2)
//glom方法
val glomRDD: RDD[Array[Int]] = listRDD.glom()
glomRDD.collect().foreach(Array => {
println(Array.mkString("-"))
})
}
}
5.对数据进行分组(groupBy)
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}
object SparkOperator04 {
def main(args: Array[String]): Unit = {
//获取sparkkconf
val conf: SparkConf = new SparkConf().setAppName("SparkOperator").setMaster("local[*]")
//获取sparkcontext
val sc = new SparkContext(conf)
//创建一个rdd
val listRDD: RDD[Int] = sc.makeRDD(List(1,2,3,4))
//对数据进行分组
val groupByRDD: RDD[(Int, Iterable[Int])] = listRDD.groupBy(x => x%2)
groupByRDD.collect().foreach(println)
}
}
6.对数据进行过滤(filter)
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}
object SparkOperator05 {
def main(args: Array[String]): Unit = {
//获取sparkkconf
val conf: SparkConf = new SparkConf().setAppName("SparkOperator").setMaster("local[*]")
//获取sparkcontext
val sc = new SparkContext(conf)
//创建一个rdd
val listRDD: RDD[Int] = sc.makeRDD(List(1,2,3,4))
//对数据进行过滤
val filterRDD: RDD[Int] = listRDD.filter(x => x%2==0)
filterRDD.collect().foreach(println)
}
}
7.以指定的随机种子随机抽样(sample)
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}
object SparkOperator06 {
def main(args: Array[String]): Unit = {
//获取sparkkconf
val conf: SparkConf = new SparkConf().setAppName("SparkOperator").setMaster("local[*]")
//获取sparkcontext
val sc = new SparkContext(conf)
//创建一个rdd
val listRDD: RDD[Int] = sc.makeRDD(1 to 10,2)
//对数据进行抽样
val samleRDD: RDD[Int] = listRDD.sample(false,0.1,1)
samleRDD.collect().foreach(println)
}
}
8.去重(distinct)
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}
object SparkOperator07 {
def main(args: Array[String]): Unit = {
//获取sparkkconf
val conf: SparkConf = new SparkConf().setAppName("SparkOperator").setMaster("local[*]")
//获取sparkcontext
val sc = new SparkContext(conf)
//创建一个rdd
val listRDD: RDD[Int] = sc.makeRDD(List(1,2,5,4,2,1,7,1,2,9,8,1,2))
//对数据进行去重
val distinctRDD: RDD[Int] = listRDD.distinct()
distinctRDD.collect().foreach(println)
}
}
9.缩减分区(coalesce)
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}
object SparkOperator08 {
def main(args: Array[String]): Unit = {
//获取sparkkconf
val conf: SparkConf = new SparkConf().setAppName("SparkOperator").setMaster("local[*]")
//获取sparkcontext
val sc = new SparkContext(conf)
//创建一个rdd
val listRDD: RDD[Int] = sc.makeRDD(1 to 16 ,4)
println("缩减分区前="+listRDD.partitions.size)
//缩减分区
val coalesceRDD: RDD[Int] = listRDD.coalesce(3)
println("缩减分区后="+coalesceRDD.partitions.size)
//distinctRDD.collect().foreach(println)
}
}
10.sortBy
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}
object SparkOperator_sortBy {
def main(args: Array[String]): Unit = {
//获取sparkkconf
val conf: SparkConf = new SparkConf().setAppName("SparkOperator").setMaster("local[*]")
//获取sparkcontext
val sc = new SparkContext(conf)
//创建一个rdd
val listRDD: RDD[Int] = sc.makeRDD(List(1,3,9,5,7,8))
//按照自身排序
val sortByRDD: RDD[Int] = listRDD.sortBy(x => x)
sortByRDD.collect().foreach(println)
}
}
二、双value类型交互
1.union
作用:对源RDD和参数RDD求并集后返回一个新的RDD
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}
object SparkOperator_union {
def main(args: Array[String]): Unit = {
//获取sparkkconf
val conf: SparkConf = new SparkConf().setAppName("SparkOperator").setMaster("local[*]")
//获取sparkcontext
val sc = new SparkContext(conf)
//创建一个rdd
val listRDD1: RDD[Int] = sc.makeRDD(1 to 10)
val listRDD2: RDD[Int] = sc.makeRDD(5 to 15)
//需求:创建两个RDD,求并集
val unionRDD: RDD[Int] = listRDD1.union(listRDD2)
unionRDD.collect().foreach(println)
}
}
2.subtract
作用:计算差的一种函数,去除两个RDD中相同的元素,不同的RDD将保留下来
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}
object SparkOperator_subtract {
def main(args: Array[String]): Unit = {
//获取sparkkconf
val conf: SparkConf = new SparkConf().setAppName("SparkOperator").setMaster("local[*]")
//获取sparkcontext
val sc = new SparkContext(conf)
//创建一个rdd
val listRDD1: RDD[Int] = sc.makeRDD(1 to 10)
val listRDD2: RDD[Int] = sc.makeRDD(3 to 8)
//需求:创建两个RDD,求第一个RDD与第二个RDD的差集
val subtractRDD: RDD[Int] = listRDD1.subtract(listRDD2)
subtractRDD.collect().foreach(println)
}
}
3.intersection
作用:对源RDD和参数RDD求交集后返回一个新的RDD
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}
object SparkOperator_intersection{
def main(args: Array[String]): Unit = {
//获取sparkkconf
val conf: SparkConf = new SparkConf().setAppName("SparkOperator").setMaster("local[*]")
//获取sparkcontext
val sc = new SparkContext(conf)
//创建一个rdd
val listRDD1: RDD[Int] = sc.makeRDD(1 to 10)
val listRDD2: RDD[Int] = sc.makeRDD(5 to 15)
//需求:创建两个RDD,求两个RDD的交集
val intersection: RDD[Int] = listRDD1.intersection(listRDD2)
intersection.collect().foreach(println)
}
}
4.cartesian
作用:笛卡尔积(尽量避免使用)
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}
object SparkOperator_cartesian {
def main(args: Array[String]): Unit = {
//获取sparkkconf
val conf: SparkConf = new SparkConf().setAppName("SparkOperator").setMaster("local[*]")
//获取sparkcontext
val sc = new SparkContext(conf)
//创建一个rdd
val listRDD1: RDD[Int] = sc.makeRDD(1 to 10)
val listRDD2: RDD[Int] = sc.makeRDD(5 to 15)
//需求:创建两个RDD,求并集
val cartesianRDD: RDD[(Int, Int)] = listRDD1.cartesian(listRDD2)
cartesianRDD.collect().foreach(println)
}
}
5.zip
作用:将两个RDD组合成Key/Value形式的RDD,这里默认两个RDD的partition数量以及元素数量都相同,否则会抛出异常。
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}
object SparkOperator_zip {
def main(args: Array[String]): Unit = {
//获取sparkkconf
val conf: SparkConf = new SparkConf().setAppName("SparkOperator").setMaster("local[*]")
//获取sparkcontext
val sc = new SparkContext(conf)
//创建一个rdd
val listRDD1: RDD[Int] = sc.makeRDD(1 to 3)
val listRDD2: RDD[String] = sc.makeRDD(List("a","b","c"))
//需求:创建两个RDD,并将两个RDD组合到一起形成一个(k,v)RDD
val zipRDD: RDD[(Int, String)] = listRDD1.zip(listRDD2)
zipRDD.collect().foreach(println)
}
}
三、Key-Value类型
1.partitionBy
作用:对pairRDD进行分区操作,如果原有的partionRDD和现有的partionRDD是一致的话就不进行分区, 否则会生成ShuffleRDD,即会产生shuffle过程。
import org.apache.spark.rdd.RDD
import org.apache.spark.{HashPartitioner, SparkConf, SparkContext}
object SparkOperator_partitionBy {
def main(args: Array[String]): Unit = {
//获取sparkkconf
val conf: SparkConf = new SparkConf().setAppName("SparkOperator").setMaster("local[*]")
//获取sparkcontext
val sc = new SparkContext(conf)
//创建一个rdd
val listRDD: RDD[(Int, String)] = sc.makeRDD(Array((1, "aaa"), (2, "bbb"), (3, "ccc"), (4, "ddd")), 4)
//需求:创建一个4个分区的RDD,对其重新分区
println(listRDD.partitions.size)
val partitionByRDD: RDD[(Int, String)] = listRDD.partitionBy(new HashPartitioner(2))
println(partitionByRDD.partitions.size)
}
}
2.groupByKey
作用:groupByKey也是对每个key进行操作,但只生成一个sequence。
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}
object SparkOperator_groupByKey {
def main(args: Array[String]): Unit = {
//获取sparkkconf
val conf: SparkConf = new SparkConf().setAppName("SparkOperator").setMaster("local[*]")
//获取sparkcontext
val sc = new SparkContext(conf)
//创建一个rdd
val arrayRDD: RDD[String] = sc.makeRDD(Array("one", "two", "two", "three", "three", "three"))
//转化数据结构,形成K-V键值对
val mapRDD: RDD[(String, Int)] = arrayRDD.map((_,1))
//根据K进行分组
val groupByKeyRDD: RDD[(String, Iterable[Int])] = mapRDD.groupByKey()
//对V进行聚合
val sumRDD: RDD[(String, Int)] = groupByKeyRDD.map {
x =>
(x._1, (x._2.sum))
}
sumRDD.collect().foreach(println)
groupByKeyRDD.collect().foreach(println)
}
}
3.reduceByKey
作用:在一个(K,V)的RDD上调用,返回一个(K,V)的RDD,使用指定的reduce函数,将相同key的值聚合到一起,reduce任务的个数可以通过第二个可选的参数来设置。
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}
object SparkOperator_reduceByKey {
def main(args: Array[String]): Unit = {
//获取sparkkconf
val conf: SparkConf = new SparkConf().setAppName("SparkOperator").setMaster("local[*]")
//获取sparkcontext
val sc = new SparkContext(conf)
//创建一个rdd
val listRDD: RDD[(String, Int)] = sc.makeRDD(List(("female",1),("male",5),("female",5),("male",2)))
// 需求:创建一个pairRDD,计算相同key对应值的相加结果
val reduceByKeyRDD: RDD[(String, Int)] = listRDD.reduceByKey {
(x, y) =>
(x + y)
}
reduceByKeyRDD.collect().foreach(println)
}
}
4.aggregateByKey
参数:(zeroValue:U,[partitioner: Partitioner]) (seqOp: (U, V) => U,combOp: (U, U) => U)
1). 作用:在kv对的RDD中,,按key将value进行分组合并,合并时,将每个value和初始值作为seq函数的参数,进行计算,返回的结果作为一个新的kv对,然后再将结果按照key进行合并,最后将每个分组的value传递给combine函数进行计算(先将前两个value进行计算,将返回结果和下一个value传给combine函数,以此类推),将key与计算结果作为一个新的kv对输出。
2). 参数描述:
(1)zeroValue:给每一个分区中的每一个key一个初始值;
(2)seqOp:函数用于在每一个分区中用初始值逐步迭代value;
(3)combOp:函数用于合并每个分区中的结果。
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}
object SparkOperator10 {
def main(args: Array[String]): Unit = {
val conf = new SparkConf().setMaster("local[*]").setAppName("SparkOperator10")
val sc = new SparkContext(conf)
//需求:创建一个pairRDD,取出每个分区相同key对应值的最大值,然后相加
val listRdd: RDD[(String, Int)] = sc.makeRDD(List(("a",3),("a",2),("c",4),("b",3),("c",6),("c",8)),2)
listRdd.glom().collect().foreach(println)
val aggregateByKeyRDD: RDD[(String, Int)] = listRdd.aggregateByKey(0)(math.max(_,_),_+_)
aggregateByKeyRDD.foreach(println)
}
}
5.foldByKey
参数:(zeroValue: V)(func: (V, V) => V): RDD[(K, V)]
作用:aggregateByKey的简化操作
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}
object SparkOperator11 {
def main(args: Array[String]): Unit = {
val conf = new SparkConf().setMaster("local[*]").setAppName("SparkOperator10")
val sc = new SparkContext(conf)
//需求:2.创建一个pairRDD,计算相同key对应值的相加结果
val listRdd: RDD[(String, Int)] = sc.makeRDD(List(("a",3),("a",2),("c",4),("b",3),("c",6),("c",8)),2)
val foldByKeyRDD: RDD[(String, Int)] = listRdd.foldByKey(0)(_+_)
foldByKeyRDD.foreach(println)
}
6.combineByKey
参数:(createCombiner: V => C, mergeValue: (C, V) => C, mergeCombiners: (C, C) => C)
- 作用:对相同K,把V合并成一个集合。
- 参数描述:
(1)createCombiner: combineByKey() 会遍历分区中的所有元素,因此每个元素的键要么还没有遇到过,要么就和之前的某个元素的键相同。如果这是一个新的元素,combineByKey()会使用一个叫作createCombiner()的函数来创建那个键对应的累加器的初始值
(2)mergeValue: 如果这是一个在处理当前分区之前已经遇到的键,它会使用mergeValue()方法将该键的累加器对应的当前值与这个新的值进行合并
(3)mergeCombiners: 由于每个分区都是独立处理的, 因此对于同一个键可以有多个累加器。如果有两个或者更多的分区都有对应同一个键的累加器, 就需要使用用户提供的 mergeCombiners() 方法将各个分区的结果进行合并。
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}
object SparkOperator12 {
def main(args: Array[String]): Unit = {
val conf = new SparkConf().setMaster("local[*]").setAppName("SparkOperator10")
val sc = new SparkContext(conf)
//需求:创建一个pairRDD,根据key计算每种key的均值。
val listRDD: RDD[(String, Int)] = sc.makeRDD(List(("a", 88), ("b", 95), ("a", 91), ("b", 93), ("a", 95), ("b", 98)), 2)
val combineByKeyRDD: RDD[(String, (Int, Int))] = listRDD.combineByKey(
(_, 1),
(acc: (Int, Int), v) => (acc._1 + v, acc._2 + 1),
(acc1: (Int, Int), acc2: (Int, Int)) => (acc1._1 + acc2._1, acc1._2 + acc2._2)
)
val value: RDD[(String, Double)] = combineByKeyRDD.map {
case (key, value) => (key, value._1 / value._2.toDouble)
}
value.foreach(println)
}
}
7.sortByKey
作用:在一个(K,V)的RDD上调用,K必须实现Ordered接口,返回一个按照key进行排序的(K,V)的RDD
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}
object SparkOperator_sortByKey {
def main(args: Array[String]): Unit = {
//获取sparkkconf
val conf: SparkConf = new SparkConf().setAppName("SparkOperator").setMaster("local[*]")
//获取sparkcontext
val sc = new SparkContext(conf)
//创建一个rdd
val arrayRDD: RDD[(Int, String)] = sc.makeRDD(Array((3, "aa"), (6, "cc"), (2, "bb"), (1, "dd")))
//需求:创建一个pairRDD,按照key的正序和倒序进行排序
val sortByKeyRDD: RDD[(Int, String)] = arrayRDD.sortByKey(true)
val sortByKeyRDD1: RDD[(Int, String)] = arrayRDD.sortByKey(false)
sortByKeyRDD.collect().foreach(println)
}
}
8.mapValues
针对于(K,V)形式的类型只对V进行操作
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}
object SparkOperator_mapValues {
def main(args: Array[String]): Unit = {
//获取sparkkconf
val conf: SparkConf = new SparkConf().setAppName("SparkOperator").setMaster("local[*]")
//获取sparkcontext
val sc = new SparkContext(conf)
//创建一个rdd
val arrayRDD: RDD[(Int, String)] = sc.makeRDD(Array((1,"a"),(2,"b"),(3,"c"),(4,"d")))
//对value添加字符串"|||"
val mapValueRDD: RDD[(Int, String)] = arrayRDD.mapValues(_+"|||")
mapValueRDD.collect().foreach(println)
}
}
9.join
作用:在类型为(K,V)和(K,W)的RDD上调用,返回一个相同key对应的所有元素对在一起的(K,(V,W))的RDD
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}
object SparkOperator_join {
def main(args: Array[String]): Unit = {
//获取sparkkconf
val conf: SparkConf = new SparkConf().setAppName("SparkOperator").setMaster("local[*]")
//获取sparkcontext
val sc = new SparkContext(conf)
//需求:创建两个pairRDD,并将key相同的数据聚合到一个元组。
val arrayRDD1: RDD[(Int, String)] = sc.makeRDD(Array((1,"a"),(2,"b"),(3,"c")))
val arrayRDD2: RDD[(Int, Int)] = sc.parallelize((Array((1,4),(2,5),(3,6))))
val joinRDD: RDD[(Int, (String, Int))] = arrayRDD1.join(arrayRDD2)
joinRDD.collect().foreach(println)
}
}