spark处理复杂的log日志文件

案例：数据

1593136280858|{"cm":{"ln":"-55.0","sv":"V2.9.6","os":"8.0.4","g":"C6816QZ0@gmail.com","mid":"489","nw":"3G","l":"es","vc":"4","hw":"640*960","ar":"MX","uid":"489","t":"1593123253541","la":"5.2","md":"sumsung-18","vn":"1.3.4","ba":"Sumsung","sr":"I"},"ap":"app","et":[{"ett":"1593050051366","en":"loading","kv":{"extend2":"","loading_time":"14","action":"3","extend1":"","type":"2","type1":"201","loading_way":"1"}},{"ett":"1593108791764","en":"ad","kv":{"activityId":"1","displayMills":"78522","entry":"1","action":"1","contentType":"0"}},{"ett":"1593111271266","en":"notification","kv":{"ap_time":"1593097087883","action":"1","type":"1","content":""}},{"ett":"1593066033562","en":"active_background","kv":{"active_source":"3"}},{"ett":"1593135644347","en":"comment","kv":{"p_comment_id":1,"addtime":"1593097573725","praise_count":973,"other_id":5,"comment_id":9,"reply_count":40,"userid":7,"content":"辑赤蹲慰鸽抿肘捎"}}]}
1593136280858|{"cm":{"ln":"-114.9","sv":"V2.7.8","os":"8.0.4","g":"NW0S962J@gmail.com","mid":"490","nw":"3G","l":"pt","vc":"8","hw":"640*1136","ar":"MX","uid":"490","t":"1593121224789","la":"-44.4","md":"Huawei-8","vn":"1.0.1","ba":"Huawei","sr":"O"},"ap":"app","et":[{"ett":"1593063223807","en":"loading","kv":{"extend2":"","loading_time":"0","action":"3","extend1":"","type":"1","type1":"102","loading_way":"1"}},{"ett":"1593095105466","en":"ad","kv":{"activityId":"1","displayMills":"1966","entry":"3","action":"2","contentType":"0"}},{"ett":"1593051718208","en":"notification","kv":{"ap_time":"1593095336265","action":"2","type":"3","content":""}},{"ett":"1593100021275","en":"comment","kv":{"p_comment_id":4,"addtime":"1593098946009","praise_count":220,"other_id":4,"comment_id":9,"reply_count":151,"userid":4,"content":"抄应螟皮釉倔掉汉蛋蕾街羡晶"}},{"ett":"1593105344120","en":"praise","kv":{"target_id":9,"id":7,"type":1,"add_time":"1593098545976","userid":8}}]}

代码：

package fuction



import org.apache.spark.SparkConf
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.types.{StructField, StructType}

/**
  * @author
  * @create
  */
object Op1 {
  def main(args: Array[String]): Unit = {
    val conf:SparkConf = new SparkConf().setAppName("kryoDemo").setMaster("local[*]")
    val spark:SparkSession = SparkSession.builder().config(conf).getOrCreate()
    val sc =spark.sparkContext

    //导入数据
    val fileRDD:RDD[String] = sc.textFile("in/tmp/op1.log")
//    fileRDD.collect.foreach(println)
    //map(x=>x.split('|'))表示按‘|’分割  map(x=>(x(0),x(1)))表示下标0和1的元素
    val jsonStrRDD = fileRDD.map(x=>x.split('|')).map(x=>(x(0),x(1)))
//    jsonStrRDD.collect.foreach(println)
    //
    val jsonRDD = jsonStrRDD.map(x=>x._2)
    jsonRDD.collect.foreach(println)

    //倒包
    import spark.implicits._
    //将jsonRDD转为DataFrame类型
    val jsonDF = jsonRDD.toDF
    //打印表结构
//    jsonDF.printSchema()
    //打印表内容
//    jsonDF.show()
    //导入Spark SQL 内置的函数
    import org.apache.spark.sql.functions._
    import org.apache.spark.sql.types._
    val jsonDF2=jsonDF.select(
      get_json_object($"value","$.cm").as("cm"),
      get_json_object($"value","$.ap").as("ap"),
      get_json_object($"value","$.et").as("et"))
//    jsonDF2.show()

    var jsonDF3=jsonDF2.select(
      get_json_object($"cm","$.ln").as("ln"),
      get_json_object($"cm","$sv").as("sv"),
      get_json_object($"cm","$.os").as("os"),
      get_json_object($"cm","$.g").as("g"),
      get_json_object($"cm","$.mid").as("nw"),
      get_json_object($"cm","$.l").as("l"),
      get_json_object($"cm","$.vc").as("vc"),
      get_json_object($"cm","$.hw").as("hw"),
      get_json_object($"cm","$.ar").as("ar"),
      get_json_object($"cm","$.uid").as("uid"),
      get_json_object($"cm","$.t").as("t"),
      get_json_object($"cm","$.la").as("la"),
      get_json_object($"cm","$.md").as("md"),
      get_json_object($"cm","$.vn").as("vn"),
      get_json_object($"cm","$.ba").as("ba"),
      get_json_object($"cm","$.sr").as("sr"),
        from_json($"et", StructType(StructField("ett", StringType)
        :: StructField("en", StringType)
        :: StructField("kv", StringType)
        :: Nil)).as("events")
    ).select($"ln",$"sv",$"os",$"g")
      jsonDF3.printSchema()
      jsonDF3.show
  }
}```
代码没写全。