1、将hadoop的配置文件:hdfs-site.xml,core-site.html文件拷贝到spark的配置文件conf下
2、将hive的配置文件:hive-site.xml拷贝到spark的配置文件conf下
3、将连接hive元数据的mysql的jar包拷贝到spark的lib目录下
4、运行spark-shell命令,进入命令行,指定集群的地址,运行内存,运行core个数,以及连接hive元数据库的JAR包
./spark-shell --master spark://*.*.*.*:7077 --executor-memory 1g --total-executor-cores 2 --driver-class-path /hadoop/work/hadoop/spark-1.6.2-bin-hadoop2.4/lib/mysql-connector-java-5.1.28.jar
5、编程程序
import org.apache.spark.sql.hive.HiveContext
val hiveContext = new HiveContext(sc)
var result = hiveContext.sql("create table person(id int,name varchar(20),age int)")
result.collect.foreach(println)
运行结果:
6、打成jar包,提交到集群运行程序
SPARK_CLASSPATH=/hadoop/work/hadoop/spark-1.6.2-bin-hadoop2.4/lib/mysql-connector-java-5.1.28.jar ./spark-submit --class SqlHive --master spark://*.*.*.*:7077 /hadoop/work/hadoop/lib/WordCount.jar
object SqlHive{ def main(args: Array[String]): Unit = { val conf = new SparkConf(); conf.setAppName("SqlHive App") val sc = new SparkContext(conf) val hsc = new HiveContext(sc) val rdd = hsc.sql("select * from stoptime limit 10") val dfdesc = rdd.describe() rdd.collect().foreach(println) }