spark:学习杂记+仿写案例--28

1.特质

def first(x: Int)
first(data)

2.操作符

.apply("Bob") //用在半生对象中，用来构造对象而不用显式的使用new

3.作为值的函数

import scala.math._
val num = 3.14
val fun = ceil _
//从技术上讲，_将ceil方法转成了函数，在scala中无法直接操纵方法，只能直接操纵函数

4.匿名函数

（x: Double） => 3 * x
Array(3.14, 1.42, 2.0).map((x: Double) => 3 * x)
//Array(9.42, 4.26, 6.0)
val fun = 3 * _  //error:无法判断出类型
val fun = 3 * (_:Double) //ok
val fun: (Double) => Double = 3 * _ //ok

5.reduceLeft方法接受一个二元的函数

   即一个带有两个参数的函数并将其应用到序列中的所有元素，从左到右

(1 to 9).reduceLeft(_ * _) //(...((1 * 2) * 3) *...* 9)
.sbortWith(_.length < _.length)

6.柯里化函数

def mulOneAtATime(x: Int)(y: Int) = x * y

package llf

import org.apache.spark.{SparkContext, SparkConf}

/**
 * Created by sendoh on 2015/4/20.
 */
object BroadcastTest {
  def main(args: Array[String]): Unit ={
    val bcName = if (args.length > 2) args(2) else "Http" //定义分块名称
    val blockSize = if (args.length > 3) args(3) else "4096" //定义分块大小
    val conf = new SparkConf().setAppName("BroadcastTest").setMaster("local[2]").setSparkHome("/usr/local/spark-1.2.0-bin-hadoop2.4")
      .set("spark.broadcast.factory", s"org.apache.spark.broadcast.${bcName}BroadcastFactory")
      .set("spark.broadcast.blockSize", blockSize)
    val sc = new SparkContext(conf)

    val slices = if (args.length > 0) args(0).toInt else 2 //分割大小
    val num = if (args.length > 1) args(1).toInt else 1000000 //分割数量

    val arr1 = (0 until num).toArray //定义arr1为0到最大分割数的数组

    for (i <- 0 until 3){ //012
      println("Iteration" + i)
      println("============")
      val startTime = System.nanoTime //定义启动时间
      val barr1 = sc.broadcast(arr1) //对arr1进行广播
      val observedSizes = sc.parallelize(1 to 10, slices).map(_ => barr1.value.size) //观测量大小，10行平行
      observedSizes.collect().foreach(i => println(i))
      println("Iteration %d took %.0f milliseconds".format(i, (System.nanoTime - startTime) / 1E6))
    }
    sc.stop()
  }

}

/

package llf
import org.apache.spark.util.Utils
import scala.collection.JavaConversions._
/**
 * Created by sendoh on 2015/4/20.
 */
object DriverSubmissionTest {
  def main(args: Array[String]): Unit ={ //args初始化形参的值
    if (args.length < 1){
      println("Usage: DriverSubmissionTest <seconds-to-sleep>")
      System.exit(0)
    }
    val numSecondsToSleep = args(0).toInt
    val env = System.getenv() //从环境中取字符串,获取环境变量的值
    val properties = Utils.getSystemProperties() //工具获取系统特性
    println("Environment variables containing SPARK_TEST:")
    env.filter{case (k, v) => k.contains("SPARK_TEST")}.foreach(println)
    println("System properties containing spark.test:")
    properties.filter{case (k, v) => k.toString.contains("spark.test")}.foreach(println)
    for (i <- 1 until numSecondsToSleep) {
      println(s"Alive for $i out of $numSecondsToSleep seconds")
      Thread.sleep(1000)
    }
  }
}