寒假第十五天

寒假第十五天

针对RDD编程

 

 

import java.util.Properties
 import org.apache.spark.sql.types._
import org.apache.spark.sql.Row
 import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.DataFrameReader
object TestMySQL {
def main(args: Array[String]) {
val spark = SparkSession.builder().appName("RddToDFrame").master("local").getOrCreate()
import spark.implicits._
val employeeRDD = spark.sparkContext.parallelize(Array("3 Mary F 26","4 Tom M 23")).map(_.split(" "))
val schema = StructType(List(StructField("id", IntegerType,true),StructField("name", StringType, true),StructField("gender", StringType,true),StructField("age", IntegerType, true)))
val rowRDD = employeeRDD.map(p => Row(p(0).toInt,p(1).trim,p(2).trim,p(3).toInt)) val employeeDF = spark.createDataFrame(rowRDD, schema)
val prop = new Properties() prop.put("user", "root") prop.put("password", "0") prop.put("driver","com.mysql.jdbc.Driver") employeeDF.write.mode("append").jdbc("jdbc:mysql://localhost:3306/sparktest","sparktest.employee", prop)
Val jdbcDF = spark.read.format("jdbc").option("url","jdbc:mysql://localhost:3306/sparktest").option("driver","com.mysql.jdbc.Driver").option("dbtable","employee").option("user","root").option("password", "0").load()
jdbcDF.agg("age" -> "max", "age" -> "sum").show()
 print("ok") } }