SparkSQL写数据到Hive的动态分区表

最新推荐文章于 2024-07-26 23:13:56 发布

Anonymous_cx

最新推荐文章于 2024-07-26 23:13:56 发布

阅读量1.2w

点赞数 1

分类专栏： Spark 文章标签： hive sparksql

本文链接：https://blog.csdn.net/Anonymous_cx/article/details/72667251

版权

Spark 专栏收录该内容

3 篇文章 0 订阅

订阅专栏

object HiveTableHelper extends Logging {

  def hiveTableInit(sc: SparkContext): HiveContext = {
    val sqlContext = new HiveContext(sc)
    sqlContext
  }

  def writePartitionTable(HCtx: HiveContext, inputRdd: RDD[Row], tabName: String, colNames: String): Unit = {
    val schema = StructType(
      colNames.split(" ").map(fieldName => StructField(fieldName, StringType, true))
    )
    val table = colNames.replace(" dt", "").split(" ").map(name => name + " String").toList.toString().replace("List(", "").replace(")", "")
    val df = HCtx.createDataFrame(inputRdd, schema)
    //df.show(5)
    //logInfo("----------------------------------begin write table-----------------------------------")
    val temptb = "temp" + tabName
    //HCtx.sql("drop table if exists " + tabName)
    df.registerTempTable(temptb)
    HCtx.sql("CREATE EXTERNAL TABLE if not exists " + tabName + " (" + table + ") PARTITIONED BY (`dt` string) ROW FORMAT DELIMITED FIELDS TERMINATED BY '|' STORED AS INPUTFORMAT  'org.apache.hadoop.mapred.SequenceFileInputFormat' OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat' location '/spdbccc/data/dest/SPSJNEW/"+tabName+"'")
    HCtx.sql("set hive.exec.dynamic.partition.mode = nonstrict")
    HCtx.sql("insert overwrite table " + tabName + " partition(`dt`)" + " select * from " + temptb)
  }
}