val spark = SourceUtil.spark
import spark.implicits._
val data = spark.sparkContext.parallelize(
Seq(Row(
Map(
"P_MOBILE_NUMBER" -> Row("86_98eesdf3434ffsfsdfs53214010708", 1665295866586L),
"P_SHA256_MOBILE_NUMBER" -> Row("86_1dsfsdfsdfdeeerrasssdfdfe66fac56e4asdfd07f5ae1", 1665295866586L),
"OTHER_INFO" -> Row("455sfsdfsfsdd", 1665295866586L)
)
))
)
val structType = StructType(Seq(StructField("value", StringType), StructField("version", LongType)))
val schema = StructType(Seq(StructField("user_params", MapType(StringType, structType))))
val subSchema = MapType(StringType, structType)
def formatPhoneNumFromHypersHma = udf((parmas: Map[String, Row]) => {
parmas.map(userInfo => {
val name = userInfo._1
val mobileNumber = userInfo._2.getAs[String]("value").toUpperCase()
val version = userInfo._2.getAs[Long]("version")
(name -> Row(mobileNumber, version))
})
}, subSchema)
val df = spark.createDataFrame(data, schema)
df.printSchema()
df
.withColumn("user_params", formatPhoneNumFromHypersHma($"user_params"))
.show(false)
```
sparkUDF处理复杂嵌套数据结构 map(string, Struct)
于 2024-01-15 14:59:25 首次发布