sparksql 操作hive_Spark sql操作Hive

这里说的是最简便的方法，通过Spark sql直接操作hive。前提是hive-site.xml等配置文件已经在Spark集群配置好。

val logger = LoggerFactory.getLogger(SevsSpark4.getClass)

def main(args: Array[String]): Unit = {

val sparkconf = new SparkConf().setAppName("sevs_spark4")

.set("HADOOP_USER_NAME", getProp("hbase.hadoop.username"))

.set("HADOOP_GROUP_NAME", getProp("hbase.hadoop.groupname"))

//  .setMaster("local")

val spark = SparkSession

.builder()

.appName("sevs spark sql")

.config(sparkconf)

.enableHiveSupport()

.getOrCreate()

// For implicit conversions like converting RDDs to DataFrames

import spark.implicits._

//读Hive示例

val df = spark.sql("select field1,field2 from table1")

df.show(1)

val count = df.count

logger.error(s"数据量={} ################",count)

//写hive示例

val wf = spark.sql("insert into table1 values ('1','201812')")

logger.error("ending sql hive#################")

}

maven依赖如下：

org.apache.spark

spark-sql_2.11

2.1.0

provided

org.apache.spark

spark-hive_2.11

2.1.0

provided