1.什么是RDD
RDD是一个容错的,只读的,可进行并行操作的数据结构,是一个分布在集群各个节点中的存放元素的集合。
2.课本p62(3.3.5)
package com.tipdm.sparkDemo import org.apache.spark.{SparkConf, SparkContext} object rz { def main(args: Array[String]): Unit = { val conf = new SparkConf().setAppName("WordCount").setMaster("local") val sc = new SparkContext(conf) val first_half = sc.textFile("D:\\Employee_salary_first_half.csv") val second_half = sc.textFile("D:\\Employee_salary_second_half.csv") val drop_first = first_half.mapPartitionsWithIndex((ix,it) =>{ if (ix == 0) it.drop(1) it }) val drop_second =second_half.mapPartitionsWithIndex((ix,it) =>{ if (ix == 0) it.drop(1) it }) val split_first = drop_first.map( line =>{val data = line.split(",");(data(1),data(6).toInt)} ) val split_second = drop_second.map( line =>{val data = line.split(",");(data(1),data(6).toInt)} ) val filter__first = split_first.filter(x => x._2 > 200000).map(x =>x._1) val filter__second = split_second.filter(x => x._2 > 200000).map(x =>x._1) val name = filter__first.union(filter__second).distinct() name.collect.foreach(println) } }
3.转换方法
package com.tipdm.sparkDemo import org.apache.spark.{SparkConf, SparkContext} object mg { def main(args: Array[String]): Unit = { val conf = new SparkConf().setAppName("WordCount").setMaster("local") val sc = new SparkContext(conf) val rdd1 = sc.parallelize(List(1,2,3)) val rdd2 = sc.parallelize(List(3,4,5,7)) val rdd3 = sc.parallelize(List('a','b','c')) val rdd4 = rdd1.union(rdd2) val rdd5 = rdd4.filter(_ >= 3) val rdd6 = rdd5.distinct() rdd6.cartesian(rdd3).collect.foreach(println) } }
4.
创建rdd1与rdd2,用union()方法合并rdd1与rdd2放在rdd4中
创建rdd5,用filter()方法过滤去除3以下的数据
创建rdd6,先用distinct()方法去重,再用cartesian()方法输出笛卡尔积
package com.tipdm.sparkDemo
import org.apache.spark.{SparkConf, SparkContext}
object a4 {
def main(args: Array[String]): Unit = {
val conf = new SparkConf().setAppName("WordCount").setMaster("local")
val sc = new SparkContext(conf)
val rdd1 = sc.parallelize(
List(('a',1),('a',2),('b',1),('c',1),('c',1))
)
val re_rdd1 = rdd1.reduceByKey((a,b) => a+b)
re_rdd1.collect.foreach(println)
val g_rdd1 = rdd1.groupByKey()
g_rdd1.collect.foreach(println)
g_rdd1.map(x => (x._1, x._2.size)).collect.foreach(println)
}
}
创建re_rdd1,用reduceByKey()方法合并统计键相同的值,将值相加输出
创建g_rdd1,用groupByKey()方法对具有相同的值进行分组,将相同键的值的数量输出
package com.tipdm.sparkDemo
import org.apache.spark.{SparkConf, SparkContext}
object a3 {
def main(args: Array[String]): Unit = {
val conf = new SparkConf().setAppName("WordCount").setMaster("local")
val sc = new SparkContext(conf)
val first_half = sc.textFile("D:\\Employee_salary_first_half.csv")
val second_half = sc.textFile("D:\\Employee_salary_first_half.csv")
val drop_first = first_half.mapPartitionsWithIndex((ix ,it) => {
if (ix == 0) it.drop(1)
it
})
val drop_second = second_half.mapPartitionsWithIndex((ix, it) => {
if (ix == 0) it.drop(1)
it
})
val split_first = drop_first.map(
line => {val data = line.split(",");(data(1), data(6).toInt)}
)
val split_second = drop_second.map(
line => {val data = line.split(",");(data(1), data(6).toInt)}
)
val filter_first = split_first.filter(x => x._2 > 200000).map(x => x._1)
val filter_second = split_second.filter(x => x._2 > 200000).map(x => x._1)
val name = filter_first.union(filter_second).distinct()
name.collect.foreach(println)
}
}
用textFile()方法将文件内容创建为rdd
用map方法加入逗号
用filter方法过滤小于20万的员工
用distinct()方法去重
最后输出结果