Spark 写数据到 hive 分区表

思路

RDD => Dataset => create partition table => create temp table ( Dataset ) =>insert

代码奉上

import org.apache.spark.SparkConf
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.{Dataset, SaveMode, SparkSession}


object Test01 {
  def main(args: Array[String]): Unit = {
  
    val conf = new SparkConf().setMaster("local[*]").setAppName("SparkSqlOnHive")
    val sparkSession = SparkSession
      .builder()
      .config(conf)
      .config("spark.sql.warehouse.dir", "hdfs://192.168.200.105:9000/user/hive/warehouse")
      .enableHiveSupport()
      .getOrCreate()
    import sparkSession.implicits._

    val dataRDD: RDD[(Int, String)] = sparkSession.sparkContext.makeRDD(List((1, "张三"), (2, "李四")))
    val studentDS: Dataset[student] = dataRDD.map {
      case (id, name) => student(id, name)
    }.toDS()
    studentDS.createOrReplaceTempView("student2021")

	sparkSession.sql("create database SkillsCompetitions")
    
    sparkSession.sql("use SkillsCompetitions")

    sparkSession.sql(
      """
        |create table student (id int, name string)
        |partitioned by (year string)
        |row format delimited fields terminated by "\t"
        |""".stripMargin)

    sparkSession.sql(
      """
        |insert into table student partition (year="2021")
        |select id, name from student2021
        |""".stripMargin)

    sparkSession.close()
  }
  case class student (id: Int, name: String)

}