思路
RDD => Dataset => create partition table => create temp table ( Dataset ) =>insert
代码奉上
import org.apache.spark.SparkConf
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.{Dataset, SaveMode, SparkSession}
object Test01 {
def main(args: Array[String]): Unit = {
val conf = new SparkConf().setMaster("local[*]").setAppName("SparkSqlOnHive")
val sparkSession = SparkSession
.builder()
.config(conf)
.config("spark.sql.warehouse.dir", "hdfs://192.168.200.105:9000/user/hive/warehouse")
.enableHiveSupport()
.getOrCreate()
import sparkSession.implicits._
val dataRDD: RDD[(Int, String)] = sparkSession.sparkContext.makeRDD(List((1, "张三"), (2, "李四")))
val studentDS: Dataset[student] = dataRDD.map {
case (id, name) => student(id, name)
}.toDS()
studentDS.createOrReplaceTempView("student2021")
sparkSession.sql("create database SkillsCompetitions")
sparkSession.sql("use SkillsCompetitions")
sparkSession.sql(
"""
|create table student (id int, name string)
|partitioned by (year string)
|row format delimited fields terminated by "\t"
|""".stripMargin)
sparkSession.sql(
"""
|insert into table student partition (year="2021")
|select id, name from student2021
|""".stripMargin)
sparkSession.close()
}
case class student (id: Int, name: String)
}