被坑了一天 （一条sql搞定）

import org.apache.spark.sql._
import scala.util.matching.Regex
import java.text.SimpleDateFormat
import java.util.Date


object UserProperty {
  def main(args: Array[String]): Unit = {
    val spark = SparkSession.builder()
      .master("local[2]")
      .appName("NPS User Property")
      .getOrCreate()

    /** 调试阶段设定loglevel=WARN */
    spark.sparkContext.setLogLevel("WARN")

    val UP_INPUT :String = "hdfs://192.168.61.165:9000/nps/NPS-立即导出-utf8.csv"
    val UP_OUTPUT :String = "hdfs://192.168.61.165:9000/nps/output/UP"

    val df: DataFrame = spark.read
      .option("header", "true")
      .csv(UP_INPUT)

      .withColumnRenamed("客户出生日期", "birthday")



    val dates: Regex = """(\d{4})-(\d{2})-(\d{2})\s(\d{2}):(\d{2}):(\d{2})""".r
    val nowdate: Long = nowDate()
    val seconds: Long = 365*24*60*60

    import spark.implicits._
    df.select("birthday").rdd.map(x =>
      x(0) match {
        case "--" => x(0)
        case dates(yy, mm, dd, hh, mi, ss) => (nowdate - s"$yy$mm$dd$hh$mi$ss".toLong)/seconds
        case _ => (nowdate - x(0).toString.toLong)/seconds
      }

    ).take(375).foreach(println)



//      df.createOrReplaceTempView("user_property")

//    spark.sql("SELECT current_date,birthday,datediff(current_date,birthday) age FROM user_property")
//      .show(350)


  }
  def nowDate(): Long = {
    val now: Date = new Date()
    val dateFormat: SimpleDateFormat = new SimpleDateFormat("yyyyMMddHHmmss")
    val date = dateFormat.format(now).toLong
    date
  }
}
case class Birthday(birthday: Long)

spark.sql("SELECT current_date,(case when birthday like '--' then '--' " +
      "when birthday like '%-% %:%' then birthday " +
      "ELSE CONCAT(substring(birthday,1,4),'-',substring(birthday,5,2),'-',substring(birthday,7,2)) END)" +
      " age FROM user_property")