spark读写Hive

最新推荐文章于 2023-09-14 15:02:16 发布

我是浣熊的微笑

最新推荐文章于 2023-09-14 15:02:16 发布

阅读量203

点赞数

本文链接：https://blog.csdn.net/gz1993/article/details/101317598

版权

package sparkUtil

import org.apache.spark.SparkConf
import org.apache.spark.sql.SparkSession

object SparkHive {

  def main(args: Array[String]): Unit = {
    val sparkConf = new SparkConf()
    sparkConf.setAppName("SparkHive")
    sparkConf.setMaster("local")
    val sparkSession = SparkSession.builder().config(sparkConf).getOrCreate()

    val birthDay = 20190919
    val DBName = ""
    val tableName = ""
    //read hive
    val readDF = sparkSession.table(DBName + "." + tableName)

    //write to hive
    val createTableSql =
      """
        |create table if not exists person (
        | id string,
        | age int
        |)
        |partitioned by (birthDay int)
        |stored as parquet
      """.stripMargin

    sparkSession.sql(createTableSql)

    val insertSql =
      s"""
         |insert overwrite table person partition(birthDay=20190919)
         |select
         |   id,age
         |from
         |   person_record
      """.stripMargin

    val writeDF = sparkSession.sql(insertSql)

    sparkSession.close()

  }
}