全量抽取到hive(静态分区)

理解分区字段

完整代码

package extract

import org.apache.spark.SparkConf
import org.apache.spark.sql.SparkSession

object test {
  def main(args: Array[String]): Unit = {
    val conf= new  SparkConf().setMaster("local[*]").setAppName("test1").set("spark.testing.memory","2147480000")
    System.setProperty("HADOOP_USER_NAME", "root")
    val connect= SparkSession.builder()
      .config("hive.metastore.uris","thrift://192.168.1.12:9083")
      .config("spark.sql.warehouse ","hdfs://192.168.1.12:9000//user/hive/warehouse")
      .config(conf).enableHiveSupport().getOrCreate()
    connect.sql("use school")

//    connect.sql("select count(*) from stu").show()
//    connect.sql("select * from stust").show()
//    connect.sql("select * from student").show()

//    连接mysql
    val connectmysql =connect.read.format("jdbc")
      .option("url","jdbc:mysql://192.168.1.12:3306/school?&useSSL=false")
      .option("driver","com.mysql.jdbc.Driver")
      .option("user","root")
      .option("password","123456")
      .option("dbtable","student").load().createOrReplaceTempView("v")
    connect.sql("select * from v").show()
//    connect.sql("drop table student")
//    将数据抽取到hive,创表
//    connect.sql(
//      """
//        |create table if not exists student(
//        |id int,
//        |name string,
//        |sex string,
//        |age string
//        |)
//        |partitioned by (aparment string)
//        |row format delimited fields terminated by ','
//        |""".stripMargin)

    //    1.dfwrite
//      .write.format("parquet") .mode("overwrite") .saveAsTable("student")

    //   2. sparksql全量抽取 静态分区
    connect.sql(
      """
        |insert overwrite
        |table school.student
        |partition(aparment="IS")
        |select id,name,sex,age from v
        |""".stripMargin)
    connect.sql("select * from school.student").show()
    connect.stop()

  }

}