spark写入hive数据

package hgs.spark.hive
import org.apache.spark.SparkConf
import org.apache.spark.SparkContext
import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.SQLContext
import org.apache.spark.sql.SaveMode
import org.apache.spark.sql.types.StructField
import org.apache.spark.sql.types.StructType
import org.apache.spark.sql.types.IntegerType
import org.apache.spark.sql.types.StringType
import org.apache.spark.sql.Row
object WriteDatatoHive {
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setAppName("WriteDatatoHive").setMaster("local")
    val context = new SparkContext(conf)
    val rdd = context.parallelize(List(("wd",22),("cm",25)), 2).map(x=>Row(x._1,x._2))
    val builder = SparkSession.builder()
           .appName("hiveApp")
           .config("spark.sql.warehouse.dir","hdfs://bigdata00:9000/user/hive/warehouse/")
           .enableHiveSupport()
           .getOrCreate()
    //import builder.implicits._
    
    
    import builder.implicits._
    
   val personShcema = StructType(
    List(
        //下面为一个列的描述，分别为 列名，数据类型，是否为空
        StructField("name",StringType,true),
        StructField("age",IntegerType,true)
     )    
    )
    
    val personDF = builder.createDataFrame(rdd, personShcema)
    personDF.createOrReplaceTempView("personm")
    
    
    //这个可以存储数据与hiveSQL兼容
    builder.table("personm").write.insertInto("test.person")
    
    //builder.sql("select * from personm").write.option("spark.sql.hive.convertMetastoreParquet", false)
     //.mode(SaveMode.Append).saveAsTable("test.person")
    
    context.stop()
        
  }
}
case class person(name:String,age:Int)

来自 “ ITPUB博客 ” ，链接：http://blog.itpub.net/31506529/viewspace-2640728/，如需转载，请注明出处，否则将追究法律责任。

转载于:http://blog.itpub.net/31506529/viewspace-2640728/