Spark udf,udaf

最新推荐文章于 2023-12-01 10:05:49 发布

Nougats

最新推荐文章于 2023-12-01 10:05:49 发布

阅读量1k

点赞数

分类专栏： spark 文章标签： spark

本文链接：https://blog.csdn.net/Nougats/article/details/77623303

版权

spark 专栏收录该内容

12 篇文章 0 订阅

订阅专栏

udf

def filterTdWithOp(operator: String): Boolean = {
    val x = ".*中国联通.*"
    val y = ".*CHN-UNICOM.*"
    val z = ".*China Unicom.*"
    if (operator!=null && operator.matches(x + "|" + y + "|" + z))
      true
    else
      false
  }
sqlContext.udf.register("filterTdWithOp", filterTdWithOp _)
td.registerTempTable("td")
val tdOp = sqlContext.sql("select *,filterTdWithOp(operator) as fiOp from td").filter("fiOp = true")

udaf

去看官网比较好

object  MaxPoint extends Aggregator[Input,Input, Input] {
override def zero: Input = Input(0,0,0,"")
  // Combine two values to produce a new value. For performance, the function may modify `buffer`
  // and return it instead of constructing a new object
  override def reduce(b: Input, a: Input): Input = {
    if(b.pointNum>a.pointNum)
      b
    else
      a
  }
  // Merge two intermediate values
  override def merge(b: Input, a: Input): Input = {
    if(b.pointNum>a.pointNum)
      b
    else
      a
  }

  override def finish(reduction: Input): Input = reduction
}