spark自定义函数UDF,处理DataFrame中列值,并返回新列。
如果只是修改列值,在withColumn(“newName”,f=>col),将将列newName替换为原始名称即可。
import org.apache.spark.sql.{DataFrame, SparkSession}
import org.apache.spark.sql.functions._
object TestSql {
def main(args: Array[String]): Unit = {
val sc: SparkSession = SparkSession.builder().master("local").appName("linShow").getOrCreate()
sc.sparkContext.setLogLevel("error")
val data = Seq(("zhang san","高级", 12), ("li si", "中级",15), ("wangwu","初级",34))
val testDF: DataFrame = sc.createDataFrame(data).toDF("name","level","num")
//定义局部函数,字符串翻转
def strReverse(src:String):String = {
if(src==null || src.isEmpty) ""
else src.reverse
}
//封装函数为sql自定义函数
var strR = udf(strReverse(_))
//调用并打印
testDF.withColumn("NewColName",strR(col("name"))).show()
}
}
打印结果如下
+---------+-----+---+----------+
| name|level|num|NewColName|
+---------+-----+---+----------+
|zhang san| 高级| 12| nas gnahz|
| li si| 中级| 15| is il|
| wangwu| 初级| 34| uwgnaw|
+---------+-----+---+----------+