@junit.Test
def mapPartitionsWithIndex(): Unit ={
sc.parallelize(Seq(1,2,3,4,5,6),2)
.mapPartitionsWithIndex((index,iter) =>{
println("index:"+index)
iter.foreach(item=>println(item))
iter
})
.collect()
}
@junit.Test
def map1(): Unit ={
sc.parallelize(Seq(1,2,3,4,5,6),2)
.mapPartitionsWithIndex((index,iter)=>{
println("index:"+index)
iter.map(item=> item *10)
iter.foreach(item => println(item))
iter
})
.collect()
}
@junit.Test
//1.定义集合
//2.过滤数据
//3.收集结果
def filter(): Unit ={ //filter相当于if结构
sc.parallelize(Seq(1,2,3,4,5,6,7,8,9,10))
.filter(item => item % 2==0)
.collect()
.foreach(item => println(item))
}
}
Spark框架—RDD算式mapPartitionsWithIndex与filter的用法
最新推荐文章于 2023-05-12 23:17:30 发布