var df = spark.sparkContext.parallelize(Array((1, "2014-11-03 10:45:58.0", 1),
(2, "2014-10-17 18:25:58.0",2),
(1, "2014-10-25 19:17:59.0",3),
(2, "2014-10-29 10:19:58.0",2),
(1, "2014-11-17 18:25:58.0",4),
(2, "2014-08-25 19:17:59.0",6),
(2, "2014-10-29 10:19:58.0",6),
(1, "2014-10-21 10:49:58.0",2))).toDF("id", "date", "target")
df = df.withColumn("date", to_date(col("date")))
val win = Window.partitionBy("id").orderBy("date").rowsBetween(-1, 0)
// 时间差要用datediff,数值可以直接相减
val df1 = df.withColumn("date_pre", datediff(col("date"), first(col("date")).over(win)))
println(df1.show())
输出:
+---+----------+------+--------+
| id| date|target|date_pre|
+---+----------+------+--------+
| 1|2014-10-21| 2| 0|
| 1|2014-10-25| 3| 4|
| 1|2014-11-03| 1| 9|
| 1|2014-11-17| 4| 14|
| 2|2014-08-25| 6| 0|
| 2|2014-10-17| 2| 53|
| 2|2014-10-29| 2| 12|
| 2|2014-10-29| 6| 0|
+---+----------+------+--------+