package com.hhb.spark.sql
import org.apache.spark.sql.{Row, SQLContext}
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.sql.types.{StructField, _}
/**
* Created by dell on 2017/7/31.
*/
object UDF {
def main(args: Array[String]) {
val conf = new SparkConf()
.setAppName("UDF")
.setMaster("local")
.set("spark.testing.memory", "2147480000")
val sc = new SparkContext(conf)
val sqlContext = new SQLContext(sc)
// 构造模拟数据
val names = Array("Leo", "Marry", "Jack", "Tom")
val namesRDD = sc.parallelize(names, 5)
val namesRowRDD = namesRDD.map{ name => Row(name) }
val structType = StructType(Array(StructField("name", StringType, true)))
val namesDF = sqlContext.createDataFrame(namesRowRDD, structType)
// 注册一张names表
namesDF.registerTempTable("names")
// 定义和注册自定义函数
// 定义函数:自己写匿名函数
// 注册函数:sqlContext.usf.register()
sqlContext.udf.register("strLen",(str: String) => str.length )
// 使用自定义函数
sqlContext.sql("select name, strLen(name) from names")
.collect()
.foreach(println)
}
}
转载于:https://my.oschina.net/hehongbo/blog/1492697