在本地提交 spark-submit
spark-sql 窗口的进入
./spark-sql --master local[4] --jars /home/hadoop/software/mysql-connector-java-5.1.27-bin.jar \
--driver-class-path /home/hadoop/software/mysql-connector-java-5.1.27-bin.jar
spark 提交任务
./bin/spark-submit \
--class test001 \
--master local \
/home/hadoop/jars/com.xx.bigdata-2.0.jar \
/home/hadoop/data/84-0.txt /home/hadoop/data/result
spark 入门案例的
import org.apache.spark.{SparkConf, SparkContext}
object test001 {
def main(args: Array[String]): Unit = {
val outpu_path=args(1)
val input_path=args(0)
val conf = new SparkConf().setAppName(this.getClass.getSimpleName).setMaster("local[4]")
val sc = new SparkContext(conf)
val data = sc.textFile(input_path)
val result = data.flatMap(_.split(" ")).map(x=>(x,1)).reduceByKey(_+_).sortBy(x=>x._2,false)
result.saveAsTextFile(outpu_path)
sc.stop()
}
}
spark 读写mysql
<dependency>
<groupId>com.typesafe</groupId>
<artifactId>config</artifactId>
<version>1.3.3</version>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>5.1.47</version>
</dependency>
读写mysql
import org.apache.spark.sql.{SaveMode, SparkSession}
import java.util.Properties
object session_source_jdbc {
def main(args: Array[String]): Unit = {
val session = SparkSession.builder().master("local[4]").appName("read_jdbc").getOrCreate()
import session.implicits._
val url="jdbc:mysql://192.168.2.123:3306/hadoop_hive?useUnicode=true&characterEncoding=UTF-8"
val table="TBLS"
val reader = session.read.format("jdbc").
option("url", url).
option("dbtable", table).
option("driver", "com.mysql.jdbc.Driver").
option("user", "root").
option("password", "root")
val frame = reader.load()
frame.createOrReplaceTempView("temp1")
val frame1 = session.sql("select TBL_ID,CREATE_TIME,OWNER from temp1 where SD_ID<=8")
frame1.show()
val url_local=" jdbc:mysql://localhost:3306/mysql001?useUnicode=true&characterEncoding=UTF-8"
val prop = new Properties()
prop.setProperty("user", "root")
prop.setProperty("password", "123456")
print("mysql 链接成功")
print("-----")
session.stop()
}
case class person(name:String,age:Int)
}