spark自定义udf函数与自定义聚合函数

udf函数

表现为对传入的数据进行处理后输出

//创建session对象
val conf = new SparkConf().setMaster("local").setAppName("jjk")
val session = SparkSession.builder().config(conf).getOrCreate()

//导入隐式转换
import session.implicits._

//在输出内容加上内容
val addName = session.udf.register("addName", (x:String)=> "Name:"+ x )

//对输入的数据进行计算
val addName = session.udf.register("avgAge", (x:Int)=>  x.toDouble / 3)

//使用
session.sql("Select addName(age) from people").show()

弱类型的自定义函数

继承UserDefinedAggregateFunction类，实现里面的方法

package com.uu.exe

import org.apache.spark.sql.Row
import org.apache.spark.sql.expressions.{MutableAggregationBuffer, UserDefinedAggregateFunction}
import org.apache.spark.sql.types._

/**
  * Created by IBM on 2020/2/26.
  */
class MyAvage extends UserDefinedAggregateFunction{

  //输入的数据结构
  override def inputSchema: StructType = {
    new StructType().add("age",LongType)
  }
//缓存，计算时的数据结构
  override def bufferSchema: StructType = {
    new StructType().add("sum",LongType).add("count",LongType)
  }
//函数返回的数据类型
  override def dataType: DataType = DoubleType
//函数的稳定性
  override def deterministic: Boolean = true
//缓冲区两个值的初始化
  override def initialize(buffer: MutableAggregationBuffer): Unit = {
    buffer(0) = 0L
    buffer(1) = 0L
  }
//更新缓冲区的数据
  override def update(buffer: MutableAggregationBuffer, input: Row): Unit = {
    buffer(0) = buffer.getLong(0) + input.getLong(0)
    buffer(1) = buffer.getLong(1) + 1
  }
//合并多个缓冲区
  override def merge(buffer1: MutableAggregationBuffer, buffer2: Row): Unit = {

//    将sum的值合并
    buffer1(0) = buffer1.getLong(0) + buffer2.getLong(0)
//    将count的值合并
    buffer1(1) = buffer1.getLong(1) + buffer2.getLong(1)

  }
//所做的具体的计算
  override def evaluate(buffer: Row): Any = {
    buffer.getLong(0).toDouble / buffer.getLong(1)
  }
}

弱类型的使用方法

//创建对象
val avg = new MyAvage
//注册函数
session.udf.register("avg",avg)
//进行查询
session.sql("select avg(avgage) from peoplel").show()

强类型的自定义函数

需要事先写出对应的样例类
使用的时候，对某个指定的列进行注册

package com.uu.exe

import org.apache.spark.sql.{Encoder, Encoders, expressions}


/**
  * Created by IBM on 2020/2/26.
  */
case class User(name:String,age:Long)
case class Avg(var sum:Long,var count:Long)

class MyAvgDs extends expressions.Aggregator[User,Avg,Double] {

  override def zero: Avg = {
    Avg(0,0)
  }
//聚合操作
  override def reduce(b: Avg, a: User): Avg ={
    b.sum = a.age + b.sum
    b.count = b.count + 1
    b
  }
//合并缓存
  override def merge(b1: Avg, b2: Avg): Avg = {
    b1.sum = b1.sum + b2.sum
    b1.count = b1.count + b2.count
    b1
  }
//返回值
  override def finish(reduction: Avg): Double = {
    reduction.sum.toDouble/reduction.count
  }
//
  override def bufferEncoder: Encoder[Avg] = Encoders.product

  override def outputEncoder: Encoder[Double] = Encoders.scalaDouble
}

使用方法

//创建对象
val ds = new MyAvgDs
//创建列名的形式
val unit = ds.toColumn.name("AvgAge")
//将ds/df进行预处理
val unit1 = frame1.as[User]
//应用函数
unit1.select(unit).show()