spark考察线程和任务的对应关系

最新推荐文章于 2022-07-13 18:37:32 发布

赵厚雄

最新推荐文章于 2022-07-13 18:37:32 发布

阅读量475

点赞数 1

本文链接：https://blog.csdn.net/nengyu/article/details/92076028

版权

import org.apache.spark.{SparkConf, SparkContext}

   /**
   * RDD再分区
   */
   object RDDRepartition {
       def main(args: Array[String]): Unit = {
           val conf = new SparkConf()
           conf.setAppName("rdddemo")
           conf.setMaster("local[3]")

           val sc = new SparkContext(conf)
           val rdd1 = sc.makeRDD((1 to 10) , 5)
           println("rdd1'pars : " + rdd1.partitions.length)
           val rdd2 = rdd1.map(e=>{
               val tid = Thread.currentThread().getId
               val tname = Thread.currentThread().getName
               printf("%d/%s : %d\r\n" , tid , tname , e)
               e
           })
           println("rdd2'pars : " + rdd2.partitions.length)
           rdd2.collect()
           while(true){
               Thread.sleep(1000)
           }
       }
   }

   //执行结果
   46/Executor task launch worker for task 0 : 1
   46/Executor task launch worker for task 0 : 2
   46/Executor task launch worker for task 3 : 7
   46/Executor task launch worker for task 3 : 8

   47/Executor task launch worker for task 1 : 3
   47/Executor task launch worker for task 1 : 4
   47/Executor task launch worker for task 4 : 9
   47/Executor task launch worker for task 4 : 10

48/Executor task launch worker for task 2 : 5
48/Executor task launch worker for task 2 : 6