集合中的常用方法
reduce方法
val arr = Array(1,2,5,7,10)
arr.reduce((x,y) => x+y) //25
arr.reduce((x,y) => x*y) //700
//reduce 默认请求是reduceLeft,从左到右开始计算
val arr2 = Array(1,2,3,4,5)
arr2.reduce(_-_) // -13 从左往右开始
arr2.reduceRight(_-_) //3 从右往左开始
fold方法
//fold默认是调用foldLeft(从左往右)
val list = List(1,2,3,4,5)
list.reduce(_+_) //15
list.fold(0)(_+_) //15
list.fold(100)(_+_) //115
list.fold("hello")(_+_) //报错
list.foldRight("hello")(_+_) //hello12345
//+可以表示字符串的拼接转换,-不可以
list.foldRight("hello")(_-_) //报错
sortwith方法
//两两比较,类似于冒泡排序
val list = List(1,4,2,-7,90,12)
list.sortwith((x,y) => x>y) // (90,12,4,2,1,-7)
list.sortwith((x,y) => x<y) //(-7,1,2,4,12,90)
flatten
val list = List(List(1,2,3,4),List(5,6,7))
list.flatten //List(1,2,3,4,5,6,7)
flatMap=flatten+Map
wordcount小案例
val lines = Array("spark hadoop hive","spark hadoop hive","hadoop hive")
//第一种方法
//切分 压平
val arr = list.flatMap(line => line.split(" "))
//按照key跟1组成元祖
val tuples = arr.map(x => (x,1))
//分组 聚合
val s = tuples.groupBy(x => x._1).map(x => (x._1,x._2.length)) //Map(spark -> 2, hadoop -> 3, hive -> 3)
//方法二
val s2 = tuples.groupBy(_._1).map(x => (x._1,x._2.map(x => x._2).sum)) //Map(spark -> 2, hadoop -> 3, hive -> 3)
//方法三
val s3 = tuples.groupBy(_._1).map(x => (x._1,x._2.foldLeft(0)(_+_._2))) //Map(spark -> 2, hadoop -> 3, hive -> 3)
方法四
val s4 = tuples.groupBy(_._1).mapValues(_.foldLeft(0)(_+_._2)) //Map(spark -> 2, hadoop -> 3, hive -> 3)
aggreate方法
val list = List(1,2,3,4,5)
//第一个参数,表示局部操作 第二个参数,表示全局操作,全局操作只是针对于分布式环境有效
list.aggregate(0)((x, y)=>x+y,(x, y)=>x+y) //15
list.aggregate(0)((x, y)=>x+y,(x, y)=>x*y) //15
list.aggregate(0)((x, y)=>x+y,(x, y)=>x-y) /15