data algorithms部分主题自写scala程序实现
SecondarySort (chapter 1)
data
2015,1,1,10
2015,1,2,11
2015,1,3,12
2015,1,4,13
2015,2,1,22
2015,2,2,23
2015,2,3,24
2015,2,4,25
2015,3,1,20
2015,3,2,21
2015,3,3,22
2015,3,4,23
2015,3,4,23
2015,1,3,12
2015,2,2,23
code
import org.apache.spark.sql.SparkSession
import org.apache.spark.{Partitioner, SparkConf}
class SecondarySortPartitioner(val v: Int) extends Partitioner {
override def numPartitions: Int = {
v
}
override def getPartition(key: Any): Int = key match {
case (k: String, v: Int) => math.abs(k.hashCode % numPartitions)
case null => 0
case _ => math.abs(key.hashCode % numPartitions)
}
}
object SecondarySort {
def main(args: Array[String]): Unit = {
System.setProperty("hadoop.home.dir", "C:\\winutils-master\\hadoop-2.6.3")
val conf = new SparkConf().setMaster("local").setAppName("SecondarySort")
val context = SparkSession.builder().config(conf).getOrCreate().sparkContext
val rdd = context.textFile("C:\\Users\\IdeaProjects\\spark_learning\\test.txt")
val step1 = rdd.map(line => line.split(","))
.map(line => ((line(0) + "-" + line(1), line(3).toInt), line(3).toInt))
val step2 = step1.repartitionAndSortWithinPartitions(new SecondarySortPartitioner(4))
.map {
case (k, v: Int) => (k._1, v.toString)
}.reduceByKey((x, y) => x + "," + y)
step2.foreach(println)
}
}
CommonFriends (chapter 8)
data
100,200 300 400 500 600
200,100 300 400
300,100 200 400 500
400,100 200 300
500,100 300
600,100
code
import org.apache.spark.{HashPartitioner, SparkConf}
import org.apache.spark.sql.SparkSession
import scala.collection.mutable.ArrayBuffer
object CommonFriends {
def main(args: Array[String]): Unit = {
System.setProperty("hadoop.home.dir", "C:\\winutils-master\\hadoop-2.6.3")
val conf = new SparkConf().setMaster("local").setAppName("CommonFriends")
val spark = SparkSession.builder().config(conf).getOrCreate()
import spark.implicits._
val context = SparkSession.builder().config(conf).getOrCreate().sparkContext
val rdd = context.textFile("C:\\Users\\IdeaProjects\\spark_learning\\test4.txt")
val step1 = rdd.map(x => x.split(",")).map(x => (x(0), x(1).split(" ")))
.repartitionAndSortWithinPartitions(new HashPartitioner(4))
.mapPartitions(
part => part.map(x => {
for {
i <- 0 until x._2.length
} yield (Seq(x._1, x._2(i)).sorted, x._2.toSeq)
})
).flatMap(xs => {
for (i <- xs.indices) yield xs(i)
})
val step2=step1.reduceByKey((x,y)=>{
val arrayBuf=new ArrayBuffer[String]()
for(j<-y){
if(x.contains(j)){
arrayBuf