1 项目配置
需要将hive-site.xml配置放在src目录。
2 代码
import org.apache.spark.SparkConf
import org.apache.spark.SparkContext
import org.apache.spark.sql.hive.HiveContext
object SparkHive {
def main(args: Array[String]): Unit = {
val conf = new SparkConf().setAppName("SparkHiveTest").
setMaster("spark://CentOS-01:7077").
set("spark.executor.memory", "4g")
val sc = new SparkContext(conf)
sc.addJar("E:\\scala_workspace\\spark\\SparkHive\\out\\artifacts\\sparkhive\\sparkhive.jar")
val hiveContext = new HiveContext(sc)
hiveContext.sql("use sogou")
val rdd = hiveContext.
sql("select count(*) from taxi_0101 a join taxi_0102 b on a.commaddr=b.commaddr")
rdd.collect().foreach(println)
sc.stop()
}
}
其中,sparkhive.jar是项目的打包jar文件。