scala常用1

1.val sc=SharedObject.sc

2.val spark = SharedObject.spark

    其中SharedObject.scala为：

    val conf =new SparkConf()

                    .setMaster("yarn")

                    .set("deploy-mode","client")

                    .set("spark.yarn.queue","yjy")

                    .set("num-excutors","5")

                    .set("executor-memory","10g")

                    .set("driver-memory","10g")

                    .set("excutor-cores","5")

val sc = new SparkContext(conf)

val spark = Spark.Session.builder().config(conf).enableHiveSupport().getOrCreate()

...

                 

3.val bd_rdd = baodan_base.map{line =>

         try{

               val eles = line.split(sep)

               val  id_num = try{get_idNum(eles(2),eles(9),eles(12),eles(5),eles(33),eles(27),eles(21),eles(17))}catch{case e:Exception =>""}

               ....

             

            (id_num,(insurancephone,insuredname,insuredphone,benficiaryname))

         }catch{case e:Exception =>("",("","","",""))}

}.filter(eles=>eles._1.length==18).map(eles => BaoDan(eles._1,eles._2._1,eles._2._2,eles._2._3,eles._2._4))

其中BaoDan为：

case class BaoDan(id_num:String,insurerphone:String,insuredname:String,insuredphone:String,beneficiaryname:String)