Spark DataFrame的Join操作和withColumn、withColumnRenamed方法实践案例（Scala Demo代码）

import org.apache.log4j.{Level, Logger}
import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.functions._

object DFTest {
  Logger.getLogger("org").setLevel(Level.ERROR)
  Logger.getRootLogger().setLevel(Level.ERROR) // 设置日志级别
  def main(args: Array[String]): Unit = {
    val spark = SparkSession.builder()
      .appName("DFTest")
      .master("local[*]")
      .getOrCreate()

    import spark.implicits._
    println("classDF")
    // 创建classDF和studentDF
    val classDF = Seq(
      (1, 1, "Alice", 90),
      (1, 2, "Bob", 80),
      (2, 1, "Charlie", 85),
      (2, 2, "David", 95)
    ).toDF("class", "no", "name", "score")

    classDF.show()
  }
}

classDF
+-----+---+-------+-----+
|class| no|   name|score|
+-----+---+-------+-----+
|    1|  1|  Alice|   90|
|    1|  2|    Bob|   80|
|    2|  1|Charlie|   85|
|    2|  2|  David|   95|
+-----+---+-------+-----+

println("studentDF")
val studentDF = Seq(
  (1, 1),
  (2, 1)
).toDF("class", "no")
studentDF.show()

studentDF
+-----+---+
|class| no|
+-----+---+
|    1|  1|
|    2|  1|
+-----+---+

println("joinedDF")
// 使用left_outer join将classDF和studentDF连接在一起
val joinedDF = classDF.join(studentDF.withColumnRenamed("class", "s_class").withColumnRenamed("no", "s_no"), $"class" === $"s_class" && $"no" === $"s_no", "left_outer")
joinedDF.show()

joinedDF
+-----+---+-------+-----+-------+----+
|class| no|   name|score|s_class|s_no|
+-----+---+-------+-----+-------+----+
|    1|  1|  Alice|   90|      1|   1|
|    1|  2|    Bob|   80|   null|null|
|    2|  1|Charlie|   85|      2|   1|
|    2|  2|  David|   95|   null|null|
+-----+---+-------+-----+-------+----+

// 添加新列flag，如果s_class和s_no在studentDF中出现，则值为1，否则为0
val resultDF = joinedDF.withColumn("flag", when(col("s_class").isNotNull && col("s_no").isNotNull, 1).otherwise(0)).drop("s_class", "s_no")
println("resultDF")
// 显示结果
resultDF.show()

resultDF
+-----+---+-------+-----+----+
|class| no|   name|score|flag|
+-----+---+-------+-----+----+
|    1|  1|  Alice|   90|   1|
|    1|  2|    Bob|   80|   0|
|    2|  1|Charlie|   85|   1|
|    2|  2|  David|   95|   0|
+-----+---+-------+-----+----+