spark dataframe 解析复杂 json

最新推荐文章于 2023-04-13 15:36:03 发布
最新推荐文章于 2023-04-13 15:36:03 发布
阅读量775 收藏 1
点赞数
分类专栏： spark
版权声明：本文为博主原创文章，遵循 CC 4.0 BY-SA 版权协议，转载请附上原文出处链接和本声明。
本文链接：https://blog.csdn.net/u010010664/article/details/107162390
版权 
package com.analysys.bacth.etl.stg

import com.analysys.logger.AnalysysLogger
import org.apache.log4j.{Level, Logger}
import org.apache.spark.sql.types._
import org.apache.spark.sql.{SparkSession, functions}

object StgDevUserNetInfo extends AnalysysLogger {

  def main(args: Array[String]): Unit = {
    Logger.getLogger("org.apache.spark").setLevel(Level.OFF)

    if (args.length < 2) {
      throw new IllegalArgumentException("Need to 2 args!!!")
    }
    val Array(proDate, topicID) = args
    val spark = SparkSession.builder().enableHiveSupport.getOrCreate()
    // 设置spark sql中查询分析器区分大小写（抽取数据时有很多异常字段，必须区分）
    spark.conf.set("spark.sql.caseSensitive", "true")
    spark.conf.set("hive.exec.dynamic.partition", "true")
    spark.conf.set("hive.exec.dynamic.partition.mode", "nonstrict")
    spark.conf.set("orc.compress", "ZLIB")
    // spark.conf.set("hive.exec.compress.output", "true")
    // spark.conf.set("mapred.output.compression.codec", "org.apache.hadoop.io.compress.GzipCodec")
    // spark.conf.set("mapred.output.compression.type", "BLOCK")
    process(spark, proDate, topicID)
    spark.sql(s"insert overwrite table stg.stg_dev_user_net_info_d partition(day='${proDate}',source='${topicID}') select DI,DB,DM,SV,SN,OSV,OSN,AVC,coalesce(x.app_key,a.AK),AC,AUI,SPN,SAN,SDKV,APIL,MO,PN,userip,MAC,str_to_map('null:null') as memo,IMSIS,AID,SNR,OUD,DBG,HJK,SIR,EGID,TMPID,IJ,IMEIS,BMAC,RES,SFS,TZ,SA,SL,SH,IR,APN,AN,ST,USM,PTL,CAD,RAD,TP from stg_dev_user_net_info_d a left join dim.dim_app_key_package_info x on a.SPN = x.package_name ")

    spark.stop()

  }

  def process(spark: SparkSession, proDate: String, topicID: String) {

    val testFlag = spark.conf.get("spark.test.flag", "0")

    import spark.implicits._
    var sqlText = "select concat(regexp_replace(source_data,'\\\\}-----',',\"HeaderInfo\":'),'}') from source.source_dev_filter WHERE day=" + proDate + " AND source=" + topicID + " AND status=0  AND get_json_object((split(source_data,'-----\\\\{')[0]),'$.NETInfo') is not null " // AND get_json_object((split(source_data,'-----\\{')[0]),'$.NETInfo') like '[{%'
    if (testFlag.equals("1")) sqlText = sqlText + " limit 1 "
    logger.info(sqlText)
    val netInfoDF = spark.sql(sqlText)
    // 初始化一条数据到rdd中，补充缺失的字段
    // 全量Json Log  val jsonLog = "{\"DevInfo\":{\"OSV\":\"null\",\"OSN\":\"null\",\"IJ\":\"null\",\"EGID\":\"null\",\"OUD\":\"null\",\"AID\":\"null\",\"PN\":\"null\",\"AUI\":\"null\",\"SN\":\"Android\",\"SV\":\"6.0.1\",\"DB\":\"OPPO\",\"DI\":\"863363036038592-null-dc6eadd31f0d0135\",\"DM\":\"OPPO R9s\",\"MAC\":\"6c:5c:14:25:be:ba\",\"SNR\":\"unknown\",\"RES\":\"1080-1920\",\"DPI\":\"480\",\"MO\":\"\",\"MPN\":\"\",\"NOC\":\"\",\"NON\":\"只能拨打紧急呼救电话\",\"IMEIS\":\"863363036038592|863363036038584\",\"IMSIS\":\"\",\"AC\":\"WanDouJia\",\"AK\":\"WanDouJia\",\"AN\":\"device\",\"APIL\":\"23\",\"APN\":\"com.device\",\"SDKV\":\"4.0.1\",\"AVC\":\"1.0|1\",\"AM\":\"49:0C:09:7A:B5:32:68:86:16:9A:D5:93:23:91:E9:EE\",\"AS\":\"F3:B6:5F:F3:EA:CC:3F:12:1F:6B:C0:DC:51:00:9F:8B:8C:FD:60:6C\",\"TMPID\":\"\",\"SIR\":\"1\",\"DBG\":\"0\",\"HJK\":\"0\",\"IR\":\"0\",\"BMAC\":\"6c:5c:14:25:be:ba\",\"BName\":\"OPPO R9s\",\"SFS\":\"1.0\",\"SH\":\"24\",\"SL\":\"zh\",\"SA\":\"CN\",\"TZ\":\"GMT+08:00\",\"SSMN\":[{\"SSN\":\"RPR0521 proximity\",\"SSV\":\"1\",\"SSM\":\"Rohm\",\"SSWUS\":true,\"SSP\":0,\"SSI\":1}],\"ETDM\":{\"BS\":\"0\",\"BH\":\"unknown\",\"BL\":\"0\",\"BSL\":\"0\",\"BP\":\"0\",\"BT\":\"unknown\",\"BTP\":\"0\",\"CM\":\"arm64-v8a:\",\"BI\":\"MMB29M\",\"BD\":\"R9s_11_A.27_180803\",\"BPT\":\"R9s\",\"BDE\":\"R9s\",\"BB\":\"msm8953\",\"BBL\":\"unknown\",\"BHW\":\"qcom\",\"BSA\":\"arm64-v8a,armeabi-v7a,armeabi\",\"BSAS\":\"armeabi-v7a,armeabi\",\"BSASS\":\"arm64-v8a\",\"BTE\":\"user\",\"BTS\":\"dev-keys\",\"BFP\":\"OPPO/R9s/R9s:6.0.1/MMB29M/1528528402:user/release-keys\",\"BRV\":\"Q_V1_P14,Q_V1_P14\",\"BIR\":\"eng.root.20181031.225852\",\"BBO\":\"OPPO/R9s/R9s:6.0.1/MMB29M/1522231161:user/release-keys\",\"BSP\":\"2018-06-05\",\"BSI\":23,\"BPSI\":0,\"BC\":\"REL\",\"IDFA\":\"\"}},\"OCInfo\":[{\"ACT\":\"1487311995660\",\"AOT\":\"1487311999662\",\"NT\":\"4G\",\"APN\":\"com.yuanding.seebaby\",\"AN\":\"掌通家园园丁\",\"AVC\":\"5.2.3|85\",\"ETDM\":{\"ST\":\"2\",\"AT\":\"SA\",\"CT\":\"2\"}},{\"ACT\":\"1487311999660\",\"AOT\":\"1487312001200\",\"NT\":\"WIFI\",\"APN\":\"com.abcpen.weike\",\"AN\":\"掌通家园园丁\",\"AVC\":\"1.0|5\",\"ETDM\":{\"ST\":\"2\",\"AT\":\"SA\",\"CT\":\"2\"}}],\"AppSnapshotInfo\":[{\"APN\":\"com.coloros.backuprestore\",\"AN\":\"备份与恢复\",\"AVC\":\"1.6.5|165\",\"AT\":\"0\",\"AHT\":\"1550410800359\"},{\"APN\":\"com.oppo.camera.facebeauty\",\"AN\":\"极致美颜\",\"AVC\":\"3.0.0_671ef32_181031|4311\",\"AT\":\"1\",\"AHT\":\"1550410800650\"},{\"APN\":\"com.oppo.camera.facebeauty\",\"AN\":\"极致美颜\",\"AVC\":\"3.0.0_671ef32_181031|4311\",\"AT\":\"2\",\"AHT\":\"1550410800650\"}],\"LocationInfo\":[{\"CT\":\"1550410800675\",\"GL\":\"40.073196-116.417301\",\"WifiInfo\":[{\"SSID\":\"HiWiFi_59A120_5G\",\"BSSID\":\"d0:ee:07:59:a1:20\",\"LEVEL\":-82,\"CBT\":\"[WPA-PSK-CCMP][WPA2-PSK-CCMP][ESS]\",\"FQC\":5220},{\"SSID\":\"\",\"BSSID\":\"34:2e:b6:ff:53:d0\",\"LEVEL\":-75,\"CBT\":\"[WPA2-PSK-CCMP][WPS][ESS]\",\"FQC\":5745},{\"SSID\":\"HUAWEI-2UXBCC\",\"BSSID\":\"34:2e:b6:cf:53:d0\",\"LEVEL\":-75,\"CBT\":\"[WPA2-PSK-CCMP][WPS][ESS]\",\"FQC\":5745}],\"BaseStationInfo\":[{\"LAC\":4432,\"CI\":24872,\"LV\":2147483647}]}],\"XXXInfo\":[\"d4bElEWTVOQ0FySUVsUFZ5QXhNeUFySUVsU1VTQXdJQ3NnVTBsU1VTQXdJRDBn\\nT0RrMkNnb2dJRkJKXG5SQ0JRVWlCRFVGVWxJRk1nSUNOVVNGSWdJQ0FnSUZaVFV5QWdJQ0FnVWxO\\nVElGQkRXU0JWU1VRZ0lDQWdJQ0JPWVcxbENqSTVPVGs0XG5JQ0EwSUNBZ05pVWdVeUFnSUNBek5T\\nQXhNakkyTkRNMlN5QWdOekU1TVRaTElDQm1aeUIxTUY5aE1UTXpJQ0JqYjIwdWNXbG9iMjh1XG5a\\nR0ZsYlc5dUNpQXhNVFUzSUNBMElDQWdNeVVnVXlBZ0lERXpOU0F5TmpNeU5qRXlTeUF4TkRrME1E\\nUkxJQ0JtWnlCemVYTjBaVzBnXG5JQ0J6ZVhOMFpXMWZjMlZ5ZG1WeUNqSTVOek0wSUNBMElDQWdN\\naVVnVXlBZ0lDQXhPU0F4Tnpjd05qTTJTeUFnTmpVM09UWkxJQ0JpXG5aeUIxTUY5aE1UUTRJQ0Jq\\nYmk1aGJtRnNlWE41Y3k1a1pXMXZPbTF2Ym1sMGIzSlRaWEoyYVdObENqSTVOell4SUNBd0lDQWdN\\nU1VnXG5VaUFnSUNBZ01TQWdNVFU0TmpoTElDQWdNVFV6TWtzZ0lHSm5JSFV3WDJFeE5EZ2dJSFJ2\\nY0FveU9UWTJOaUFnTlNBZ0lERWxJRk1nXG5JQ0FnTXprZ01qQXhNVGMwTkVzZ0lEYzFNelF3U3lB\\nZ1ptY2dkVEJmWVRFME9DQWdZMjR1WVc1aGJIbHplWE11WkdWdGJ3b2dJRE00XG5OaUFnTUNBZ0lE\\nRWxJRk1nSUNBZ016\"],\"NETInfo\":[{\"AN\":\"支付宝\",\"APN\":\"com.eg.android.AlipayGphone\",\"SINF\":[{\"CONNS\":[{\"CAD\":\"192.168.6.199:47178\",\"PTL\":\"tcp\",\"RAD\":\"110.76.15.39:443\",\"TP\":\"ESTABLISHED\"},{\"CAD\":\"::FFFF:0:192.168.6.199\\\\48896\",\"PTL\":\"tcp6\",\"RAD\":\"::FFFF:0:110.75.134.5\\\\443\",\"TP\":\"CLOSE_WAIT\"},{\"CAD\":\"192.168.6.199:45817\",\"PTL\":\"udp\",\"RAD\":\"203.209.245.101:443\",\"TP\":\"ESTABLISHED\"}],\"ST\":1573726492324,\"USM\":\"com.device\"}]}],\"HeaderInfo\":{\"userip\":\"39.177.105.95\",\"AK\":\"8752144198607958d\",\"SDKV\":\"4.3.0.4|20191015\",\"OSV\": \"\",\"tmpid\":\"\",\"egid\":\"\",\"time\": \"1591005283727\",\"debug\": \"0\"}}"
    // 初始化一条默认的Json 获取标准schema
    val jsonLog = "{\"DevInfo\":{\"OSV\":\"null\",\"OSN\":\"null\",\"IJ\":\"null\",\"EGID\":\"null\",\"OUD\":\"null\",\"AID\":\"null\",\"PN\":\"null\",\"AUI\":\"null\",\"SN\":\"Android\",\"SV\":\"6.0.1\",\"DB\":\"OPPO\",\"DI\":\"863363036038592-null-dc6eadd31f0d0135\",\"DM\":\"OPPO R9s\",\"MAC\":\"6c:5c:14:25:be:ba\",\"SNR\":\"unknown\",\"RES\":\"1080-1920\",\"DPI\":\"480\",\"MO\":\"\",\"MPN\":\"\",\"NOC\":\"\",\"NON\":\"只能拨打紧急呼救电话\",\"IMEIS\":\"863363036038592|863363036038584\",\"IMSIS\":\"\",\"AC\":\"WanDouJia\",\"AK\":\"WanDouJia\",\"AN\":\"device\",\"APIL\":\"23\",\"APN\":\"com.device\",\"SDKV\":\"4.0.1\",\"AVC\":\"1.0|1\",\"AM\":\"49:0C:09:7A:B5:32:68:86:16:9A:D5:93:23:91:E9:EE\",\"AS\":\"F3:B6:5F:F3:EA:CC:3F:12:1F:6B:C0:DC:51:00:9F:8B:8C:FD:60:6C\",\"TMPID\":\"\",\"SIR\":\"1\",\"DBG\":\"0\",\"HJK\":\"0\",\"IR\":\"0\",\"BMAC\":\"6c:5c:14:25:be:ba\",\"BName\":\"OPPO R9s\",\"SFS\":\"1.0\",\"SH\":\"24\",\"SL\":\"zh\",\"SA\":\"CN\",\"TZ\":\"GMT+08:00\"},\"NETInfo\":[{\"AN\":\"支付宝\",\"APN\":\"com.eg.android.AlipayGphone\",\"SINF\":[{\"CONNS\":[{\"CAD\":\"192.168.6.199:47178\",\"PTL\":\"tcp\",\"RAD\":\"110.76.15.39:443\",\"TP\":\"ESTABLISHED\"},{\"CAD\":\"::FFFF:0:192.168.6.199\\\\48896\",\"PTL\":\"tcp6\",\"RAD\":\"::FFFF:0:110.75.134.5\\\\443\",\"TP\":\"CLOSE_WAIT\"},{\"CAD\":\"192.168.6.199:45817\",\"PTL\":\"udp\",\"RAD\":\"203.209.245.101:443\",\"TP\":\"ESTABLISHED\"}],\"ST\":1573726492324,\"USM\":\"com.device\"}]}],\"HeaderInfo\":{\"userip\":\"39.177.105.95\",\"AK\":\"8752144198607958d\",\"SDKV\":\"4.3.0.4|20191015\",\"OSV\": \"\",\"tmpid\":\"\",\"egid\":\"\",\"time\": \"1591005283727\",\"debug\": \"0\"}}"
    val jsonLogData = Array(jsonLog)
    // parallelize(data, nums)中nums是设置分片数量的，默认情况下spark会自动根据集群情况进行设定的
    val jsonLogDataRDD = spark.sparkContext.parallelize(jsonLogData, 1)
    val jsonLogDataSchema = spark.read.json(jsonLogDataRDD).schema
    jsonLogDataSchema.printTreeString()
    //防止脏数据产生异常，使用标准schema解析
    val netInfoJsonDF = spark.read.schema(jsonLogDataSchema).json(netInfoDF.rdd.map(_.toString()))
    netInfoJsonDF.printSchema
    /**
      * root
      * |-- AppSnapshotInfo: array (nullable = true)
      * |    |-- element: struct (containsNull = true)
      * |    |    |-- AHT: string (nullable = true)
      * |    |    |-- AN: string (nullable = true)
      * |    |    |-- APN: string (nullable = true)
      * |    |    |-- AT: string (nullable = true)
      * |    |    |-- AVC: string (nullable = true)
      * |-- DevInfo: struct (nullable = true)
      * |    |-- AC: string (nullable = true)
      * |    |-- AID: string (nullable = true)
      * |    |-- AK: string (nullable = true)
      * |    |-- AM: string (nullable = true)
      * |    |-- AN: string (nullable = true)
      * |    |-- APIL: string (nullable = true)
      * |    |-- APN: string (nullable = true)
      * |    |-- AS: string (nullable = true)
      * |    |-- AUI: string (nullable = true)
      * |    |-- AVC: string (nullable = true)
      * |    |-- BMAC: string (nullable = true)
      * |    |-- BName: string (nullable = true)
      * |    |-- DB: string (nullable = true)
      * |    |-- DBG: string (nullable = true)
      * |    |-- DI: string (nullable = true)
      * |    |-- DM: string (nullable = true)
      * |    |-- DPI: string (nullable = true)
      * |    |-- EGID: string (nullable = true)
      * |    |-- ETDM: struct (nullable = true)
      * |    |    |-- BB: string (nullable = true)
      * |    |    |-- BBL: string (nullable = true)
      * |    |    |-- BBO: string (nullable = true)
      * |    |    |-- BC: string (nullable = true)
      * |    |    |-- BD: string (nullable = true)
      * |    |    |-- BDE: string (nullable = true)
      * |    |    |-- BFP: string (nullable = true)
      * |    |    |-- BH: string (nullable = true)
      * |    |    |-- BHW: string (nullable = true)
      * |    |    |-- BI: string (nullable = true)
      * |    |    |-- BIR: string (nullable = true)
      * |    |    |-- BL: string (nullable = true)
      * |    |    |-- BP: string (nullable = true)
      * |    |    |-- BPSI: string (nullable = true)
      * |    |    |-- BPT: string (nullable = true)
      * |    |    |-- BRV: string (nullable = true)
      * |    |    |-- BS: string (nullable = true)
      * |    |    |-- BSA: string (nullable = true)
      * |    |    |-- BSAS: string (nullable = true)
      * |    |    |-- BSASS: string (nullable = true)
      * |    |    |-- BSI: string (nullable = true)
      * |    |    |-- BSL: string (nullable = true)
      * |    |    |-- BSP: string (nullable = true)
      * |    |    |-- BT: string (nullable = true)
      * |    |    |-- BTE: string (nullable = true)
      * |    |    |-- BTP: string (nullable = true)
      * |    |    |-- BTS: string (nullable = true)
      * |    |    |-- CM: string (nullable = true)
      * |    |    |-- IDFA: string (nullable = true)
      * |    |-- HJK: string (nullable = true)
      * |    |-- IJ: string (nullable = true)
      * |    |-- IMEIS: string (nullable = true)
      * |    |-- IMSIS: string (nullable = true)
      * |    |-- IR: string (nullable = true)
      * |    |-- MAC: string (nullable = true)
      * |    |-- MO: string (nullable = true)
      * |    |-- MPN: string (nullable = true)
      * |    |-- NOC: string (nullable = true)
      * |    |-- NON: string (nullable = true)
      * |    |-- OSN: string (nullable = true)
      * |    |-- OSV: string (nullable = true)
      * |    |-- OUD: string (nullable = true)
      * |    |-- PN: string (nullable = true)
      * |    |-- RES: string (nullable = true)
      * |    |-- SA: string (nullable = true)
      * |    |-- SDKV: string (nullable = true)
      * |    |-- SFS: string (nullable = true)
      * |    |-- SH: string (nullable = true)
      * |    |-- SIR: string (nullable = true)
      * |    |-- SL: string (nullable = true)
      * |    |-- SN: string (nullable = true)
      * |    |-- SNR: string (nullable = true)
      * |    |-- SSMN: array (nullable = true)
      * |    |    |-- element: struct (containsNull = true)
      * |    |    |    |-- SSI: long (nullable = true)
      * |    |    |    |-- SSM: string (nullable = true)
      * |    |    |    |-- SSN: string (nullable = true)
      * |    |    |    |-- SSP: long (nullable = true)
      * |    |    |    |-- SSV: string (nullable = true)
      * |    |    |    |-- SSWUS: boolean (nullable = true)
      * |    |-- SV: string (nullable = true)
      * |    |-- TMPID: string (nullable = true)
      * |    |-- TZ: string (nullable = true)
      * |-- HeaderInfo: struct (nullable = true)
      * |    |-- AK: string (nullable = true)
      * |    |-- OSV: string (nullable = true)
      * |    |-- SDKV: string (nullable = true)
      * |    |-- debug: string (nullable = true)
      * |    |-- egid: string (nullable = true)
      * |    |-- time: string (nullable = true)
      * |    |-- tmpid: string (nullable = true)
      * |    |-- userip: string (nullable = true)
      * |-- LocationInfo: array (nullable = true)
      * |    |-- element: struct (containsNull = true)
      * |    |    |-- BaseStationInfo: array (nullable = true)
      * |    |    |    |-- element: struct (containsNull = true)
      * |    |    |    |    |-- CI: long (nullable = true)
      * |    |    |    |    |-- LAC: long (nullable = true)
      * |    |    |    |    |-- LV: long (nullable = true)
      * |    |    |-- CT: string (nullable = true)
      * |    |    |-- GL: string (nullable = true)
      * |    |    |-- WifiInfo: array (nullable = true)
      * |    |    |    |-- element: struct (containsNull = true)
      * |    |    |    |    |-- BSSID: string (nullable = true)
      * |    |    |    |    |-- CBT: string (nullable = true)
      * |    |    |    |    |-- FQC: long (nullable = true)
      * |    |    |    |    |-- LEVEL: long (nullable = true)
      * |    |    |    |    |-- SSID: string (nullable = true)
      * |-- NETInfo: array (nullable = true)
      * |    |-- element: struct (containsNull = true)
      * |    |    |-- AN: string (nullable = true)
      * |    |    |-- APN: string (nullable = true)
      * |    |    |-- SINF: array (nullable = true)
      * |    |    |    |-- element: struct (containsNull = true)
      * |    |    |    |    |-- CONNS: array (nullable = true)
      * |    |    |    |    |    |-- element: struct (containsNull = true)
      * |    |    |    |    |    |    |-- CAD: string (nullable = true)
      * |    |    |    |    |    |    |-- PTL: string (nullable = true)
      * |    |    |    |    |    |    |-- RAD: string (nullable = true)
      * |    |    |    |    |    |    |-- TP: string (nullable = true)
      * |    |    |    |    |-- ST: long (nullable = true)
      * |    |    |    |    |-- USM: string (nullable = true)
      * |-- OCInfo: array (nullable = true)
      * |    |-- element: struct (containsNull = true)
      * |    |    |-- ACT: string (nullable = true)
      * |    |    |-- AN: string (nullable = true)
      * |    |    |-- AOT: string (nullable = true)
      * |    |    |-- APN: string (nullable = true)
      * |    |    |-- AVC: string (nullable = true)
      * |    |    |-- ETDM: struct (nullable = true)
      * |    |    |    |-- AT: string (nullable = true)
      * |    |    |    |-- CT: string (nullable = true)
      * |    |    |    |-- ST: string (nullable = true)
      * |    |    |-- NT: string (nullable = true)
      * |-- XXXInfo: array (nullable = true)
      * |    |-- element: string (containsNull = true)
      */
    // flat data
    val net_info = netInfoJsonDF.select($"DevInfo.DI", $"DevInfo.DB", $"DevInfo.DM", $"DevInfo.SV", $"DevInfo.SN", $"DevInfo.OSV", $"DevInfo.OSN", $"DevInfo.AVC", $"DevInfo.AK", $"DevInfo.AC", $"DevInfo.AUI", $"DevInfo.APN".as("SPN"), $"DevInfo.AN".as("SAN"), $"DevInfo.SDKV", $"DevInfo.APIL", $"DevInfo.MO", $"DevInfo.PN", $"HeaderInfo.userip", $"DevInfo.MAC", $"DevInfo.IMSIS", $"DevInfo.AID", $"DevInfo.SNR", $"DevInfo.OUD", $"DevInfo.DBG", $"DevInfo.HJK", $"DevInfo.SIR", $"DevInfo.EGID", $"DevInfo.TMPID", $"DevInfo.IJ", $"DevInfo.IMEIS", $"DevInfo.BMAC", $"DevInfo.RES", $"DevInfo.SFS", $"DevInfo.TZ", $"DevInfo.SA", $"DevInfo.SL", $"DevInfo.SH", $"DevInfo.IR", functions.explode($"NETInfo"))
    val net_info_sinf = net_info.select($"DI", $"DB", $"DM", $"SV", $"SN", $"OSV", $"OSN", $"AVC", $"AK", $"AC", $"AUI", $"SPN", $"SAN", $"SDKV", $"APIL", $"MO", $"PN", $"userip", $"MAC", $"IMSIS", $"AID", $"SNR", $"OUD", $"DBG", $"HJK", $"SIR", $"EGID", $"TMPID", $"IJ", $"IMEIS", $"BMAC", $"RES", $"SFS", $"TZ", $"SA", $"SL", $"SH", $"IR", $"col.APN", $"col.AN", functions.explode($"col.SINF"))
    val net_info_conns = net_info_sinf.select($"DI", $"DB", $"DM", $"SV", $"SN", $"OSV", $"OSN", $"AVC", $"AK", $"AC", $"AUI", $"SPN", $"SAN", $"SDKV", $"APIL", $"MO", $"PN", $"userip", $"MAC", $"IMSIS", $"AID", $"SNR", $"OUD", $"DBG", $"HJK", $"SIR", $"EGID", $"TMPID", $"IJ", $"IMEIS", $"BMAC", $"RES", $"SFS", $"TZ", $"SA", $"SL", $"SH", $"IR", $"APN", $"AN", $"col.ST", $"col.USM", functions.explode($"col.CONNS"))
    val stg_dev_user_net_info_d = net_info_conns.selectExpr("DI", "DB", "DM", "SV", "SN", "OSV", "OSN", "AVC", "AK", "AC", "AUI", "SPN", "SAN", "SDKV", "APIL", "MO", "PN", "userip", "MAC", "IMSIS", "AID", "SNR", "OUD", "DBG", "HJK", "SIR", "EGID", "TMPID", "IJ", "IMEIS", "BMAC", "RES", "SFS", "TZ", "SA", "SL", "SH", "IR", "APN", "AN", "ST", "USM", "col.PTL", "col.CAD", "col.RAD", "col.TP").where("TP in ('ESTABLISHED', 'LISTEN', 'SYN_SENT', 'SYN_RECV')")
    stg_dev_user_net_info_d.createOrReplaceTempView("stg_dev_user_net_info_d")

    if (testFlag.equals("1")) {
      netInfoDF.show
      netInfoJsonDF.show
      stg_dev_user_net_info_d.show
    }

  }
}

 

听见下雨的声音hb
关注
  • 0
    点赞
  • 踩
  • 1
    收藏
    觉得还不错? 一键收藏
  • 0
    评论
专栏目录
pandas和spark dataframe互相转换实例详解
09-17
主要介绍了pandas和spark dataframe互相转换实例详解,文中通过示例代码介绍的非常详细，对大家的学习或者工作具有一定的参考学习价值,需要的朋友可以参考下
dataframe pandas 取得当前时间段的后一个时间点数据
a5601564的博客
09-28 2152
数据从星期一到星期五的日线数据形成周线数据，然后想定位到所对应的周线数据的情况，就可以通过 backfill参数实现。 data_day_df.index.get_loc(current_kline.open_time, method='backfill') open_time为当前的日线级数据。运算后得到周线数据。 ...
python —— f r u等关键字
独孤东方朔的博客
11-25 1439
字符串前加 r "r"的作用是去除转义字符. 即如果是“\n”那么表示一个反斜杠字符，一个字母n，而不是表示换行了。 print('test\n') print(r'test\n') 字符串前加 f import time t0 = time.time() time.sleep(1) name = 'processing' # 以 f开头表示在字符串内支持大括号内的python 表达 print(f'{name} done in {time.time() - t0:.2f} s') 输出： pr
Spark 之 解析json的复杂和嵌套数据结构
m0_73469590的博客
04-11 1210
Spark 之 解析json的复杂和嵌套数据结构
大数据技术之Spark SQL——解析JSON字符串
five小点心的博客
04-11 1399
sc.textFile读取数据源，并对结构化数据进行拆分。
Spark操作 对json复杂和嵌套数据结构的操作
最新发布
weixin_53898747的博客
04-13 607
从Json取值-----get_json_object，to_json()将获取的数据转化为json格式，对嵌套结构的处理（样例类和炸裂函数使用来解决）。
Spark DataFrame详解.zip
05-29
Spark DataFrame使用详解，包括：DataFrame解析；创建；Action;条件查询和join详尽操作解释
Spark DataFrame 演示Demo
05-20
简单一个示例，演示Spark中DataFrame的创建与操作
Spark学习笔记（三）：Spark DataFrame
01-20
Spark SQL增加了DataFrame（即带有Schema信息的RDD），使用户可以在Spark SQL中执行SQL语句，数据既可以来自RDD，也可以是Hive、HDFS、Cassandra等外部数据源，还可以是JSON格式的数据 Spark SQL目前支持Scala、...
Spark DataFrame
03-07
详细介绍了基于RDD的DataFrame数据结构以及操作接口。
Spark SQL操作JSON字段的小技巧
09-09
主要给大家介绍了关于Spark SQL操作JSON字段的小技巧，文中通过示例代码介绍的非常详细，对大家学习或者使用spark sql具有一定的参考学习价值，需要的朋友们下面随着小编来一起学习学习吧。
spark 输出结果压缩（gz）
热门推荐
跟我一起去征服
07-07 1万+
如果不想往下看，可以直接看结果：maxCallRdd.repartition(3).saveAsTextFile(path,GzipCodec.class); 恩，没错。就只这么一行简单的代码实现了gz压缩，但是为什么网上一直没找到呢，这个我不太清楚， 可能是他们没碰到吧。     最近项目的需求，其实很简单，就是将输出结果以.gz的格式压缩，每个压缩包的大小不能超过100M,而且压缩包的名
Spark之简介
TSzero的博客
07-13 360
Spark计算平台简介
spark 任务调优参数及问题整理
小小码农的博客
09-01 1937
1.数据倾斜： 最简单的做法:在sql中使用随机数distribute by rand() 在sql 末尾加 select * from a left join b on a.xx=b.xx distribute by rand() 2.spark.dynamicAllocation.enabled=true开启动态资源分配 3.提高并行度 spark.sql.shuffle.partitions=1000 ， 调整stage的并行度，也就是每个stage的task个数，默认值...
spark解析复杂json，json内结构为主表嵌套多个从表，每个从表中又嵌套表，嵌套层级有的在5次以上。Array嵌套Array、每条json的字段数量不一定。。。。
枪枪枪的博客
02-16 1017
这个json字符串很长，结构也很复杂，手工构造struct的话不太现实 开始 一开始按如下的方式对数据进行解析 zip文件–》RDD–〉spark.read.json(dataRDD)–〉DataFrame 通过spark.sql( … ,key1.key2.key3 ,…) 可以取到[key1.key2.key3]下的值 但是这个方法有个缺点，对于Array类型的节点，只能取一次，无法循环获取Array内的所有元素 于是准备借助其它json包对RDD内的jsonString进行解析，然后对Array类型.
spark中dataframe解析_Spark DataFrame解析json字符串
weixin_39701834的博客
12-21 1424
近期在做Spark Streaming方面的测试，从Kafka中实时取数据。此时接收到的数据是一段 json 数组形式的字符串，那么就需要将其解析为DataFrame的行数据，以方便进行实时运算。下面的代码示例演示了如何实现这个功能，因为比较简单，就多作说明了。假设初始的字符串是：[{"a":1,"b":2},{"a":3,"b":4},{"a":5,"b":6},{"a":7,"b":8}]，最...
Spark的Dataframe（Dataset）转Json
tianhouquan的专栏
12-29 6461
                       Spark的Dataframe（Dataset）转Json字符串   Spark的Dataframe（Dataset）转Json,网上很多都说的太麻烦了，让你先转换成collect，然后按照不同的filed字段去转义。 完全不用那么麻烦 原理就是先把Dataset&lt;row&gt;就是Dataframe对象（带schema的数据类型）转换成...
【Spark】使用DataFrame读取复杂JSON中的嵌套数组
Big data enthusiast
06-01 3575
所周知，在早期Spark版本中就已经支持读取Json格式的数据文件，并能够直接转换为数据库表，以方便我们进行处理数据，在本篇文章中我们将介绍如何通过Spark API很简...

“相关推荐”对你有帮助么？

  • 非常没帮助
  • 没帮助
  • 一般
  • 有帮助
  • 非常有帮助
提交
评论
添加红包

请填写红包祝福语或标题

个

红包个数最小为10个

元

红包金额最低5元

当前余额3.43元 前往充值 >
需支付：10.00元
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付元
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明：

1.余额是钱包充值的虚拟货币，按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载，可以购买VIP、付费专栏及课程。

余额充值