数据算法--Hadoop/Spark大数据处理技巧

data algorithms部分主题自写scala程序实现

---

SecondarySort (chapter 1)

data

2015,1,1,10
2015,1,2,11
2015,1,3,12
2015,1,4,13
2015,2,1,22
2015,2,2,23
2015,2,3,24
2015,2,4,25
2015,3,1,20
2015,3,2,21
2015,3,3,22
2015,3,4,23
2015,3,4,23
2015,1,3,12
2015,2,2,23

code

import org.apache.spark.sql.SparkSession
import org.apache.spark.{Partitioner, SparkConf}


class SecondarySortPartitioner(val v: Int) extends Partitioner {
  override def numPartitions: Int = {
    v
  }

  override def getPartition(key: Any): Int = key match {
    case (k: String, v: Int) => math.abs(k.hashCode % numPartitions)
    case null => 0
    case _ => math.abs(key.hashCode % numPartitions)
  }
}


object SecondarySort {
  def main(args: Array[String]): Unit = {
    System.setProperty("hadoop.home.dir", "C:\\winutils-master\\hadoop-2.6.3")
    val conf = new SparkConf().setMaster("local").setAppName("SecondarySort")
    val context = SparkSession.builder().config(conf).getOrCreate().sparkContext
    val rdd = context.textFile("C:\\Users\\IdeaProjects\\spark_learning\\test.txt")
    val step1 = rdd.map(line => line.split(","))
      .map(line => ((line(0) + "-" + line(1), line(3).toInt), line(3).toInt))
    val step2 = step1.repartitionAndSortWithinPartitions(new SecondarySortPartitioner(4))
      .map {
        case (k, v: Int) => (k._1, v.toString)
      }.reduceByKey((x, y) => x + "," + y)
    step2.foreach(println)
  }

}

---

CommonFriends (chapter 8)

data

100,200 300 400 500 600
200,100 300 400
300,100 200 400 500
400,100 200 300
500,100 300
600,100

code

import org.apache.spark.{HashPartitioner, SparkConf}
import org.apache.spark.sql.SparkSession

import scala.collection.mutable.ArrayBuffer

object CommonFriends {
  def main(args: Array[String]): Unit = {
    System.setProperty("hadoop.home.dir", "C:\\winutils-master\\hadoop-2.6.3")
    val conf = new SparkConf().setMaster("local").setAppName("CommonFriends")
    val spark = SparkSession.builder().config(conf).getOrCreate()
    import spark.implicits._
    val context = SparkSession.builder().config(conf).getOrCreate().sparkContext
    val rdd = context.textFile("C:\\Users\\IdeaProjects\\spark_learning\\test4.txt")
    val step1 = rdd.map(x => x.split(",")).map(x => (x(0), x(1).split(" ")))
      .repartitionAndSortWithinPartitions(new HashPartitioner(4))
      .mapPartitions(
        part => part.map(x => {
          for {
            i <- 0 until x._2.length
          } yield (Seq(x._1, x._2(i)).sorted, x._2.toSeq)
        })
      ).flatMap(xs => {
      for (i <- xs.indices) yield xs(i)
    })
    val step2=step1.reduceByKey((x,y)=>{
      val arrayBuf=new ArrayBuffer[String]()
        for(j<-y){
          if(x.contains(j)){
            arrayBuf