spark+shell连接mysql_spark SQL学习（spark连接 mysql）

spark连接mysql(打jar包方式)

package wujiadong_sparkSQL

import java.util.Properties

import org.apache.spark.sql.SQLContext

import org.apache.spark.{SparkConf, SparkContext}

/**

* Created by Administrator on 2017/2/14.

*/

object JdbcOperation {

def main(args: Array[String]): Unit = {

val conf = new SparkConf().setAppName("JdbcOperation")

val sc = new SparkContext(conf)

val sqlContext = new SQLContext(sc)

val properties = new Properties()

properties.put("user","feigu")

properties.put("password","feigu")

val url = "jdbc:mysql://slave02:3306/testdb?useUnicode=true&characterEncoding=gbk&zeroDateTimeBehavior=convertToNull"

val stud_scoreDF = sqlContext.read.jdbc(url,"stud_score",properties)

stud_scoreDF.show()

}

}

提交集群

hadoop@master:~/wujiadong$ spark-submit --driver-class-path /home/hadoop/bigdata/hive/lib/mysql-connector-java-5.1.10-2.jar --class wujiadong_sparkSQL.JdbcOperation --executor-memory 500m --total-executor-cores 2 /home/hadoop/wujiadong/wujiadong.spark.jar

或者

hadoop@master:~/wujiadong$ spark-submit --jars /home/hadoop/bigdata/hive/lib/mysql-connector-java-5.1.10-2.jar --class wujiadong_sparkSQL.JdbcOperation --executor-memory 500m --total-executor-cores 2 /home/hadoop/wujiadong/wujiadong.spark.jar

运行结果

hadoop@master:~/wujiadong$ spark-submit --driver-class-path /home/hadoop/bigdata/hive/lib/mysql-connector-java-5.1.10-2.jar --class wujiadong_sparkSQL.JdbcOperation --executor-memory 500m --total-executor-cores 2 /home/hadoop/wujiadong/wujiadong.spark.jar

17/02/15 13:21:06 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable

17/02/15 13:21:09 INFO Slf4jLogger: Slf4jLogger started

17/02/15 13:21:09 INFO Remoting: Starting remoting

17/02/15 13:21:09 INFO Remoting: Remoting started; listening on addresses :[akka.tcp://sparkDriver@192.168.1.131:40654]

17/02/15 13:21:13 WARN MetricsSystem: Using default name DAGScheduler for source because spark.app.id is not set.

+----------+--------+--------+--------+---------+---------+

| stud_code|sub_code|sub_name|sub_tech|sub_score|stat_date|

+----------+--------+--------+--------+---------+---------+

|2015101000| 10101| 数学分析| | 90| null|

|2015101000| 10102| 高等代数| | 88| null|

|2015101000| 10103| 大学物理| | 67| null|

|2015101000| 10104| 计算机原理| | 78| null|

|2015101000| 10105| 电磁学| | 89| null|

|2015101001| 10101| 数学分析| | 87| null|

|2015101001| 10102| 高等代数| | 78| null|

|2015101001| 10103| 大学物理| | 88| null|

|2015101001| 10104| 计算机原理| | 86| null|</