Spark2.X之scala JDBCDataSource

/**

1.获取MySQL数据库中学生基本信息表与学生成绩表

2.通过join操作与filter操作将学生分数大于等于80分的学生存入好学生的表中。

**/

import org.apache.spark.sql.SparkSession
import org.apache.commons.collections.map.HashedMap
import java.util.Properties
import org.apache.spark.sql.Row
import org.apache.spark.sql.types.{StructType,StructField,StringType};
import org.apache.spark.sql.types.IntegerType
/**
 * @author Administrator
 */
object  JDBCDataSource {
  
  def main(args: Array[String]): Unit = {
    
    val spark = SparkSession.
      builder().
      appName("JDBCDataSource").
      master("local").
      getOrCreate() 
      val connectionProperties = new Properties()
      connectionProperties.put("user", "root")
      connectionProperties.put("password", "root")
      val student_infos = spark.read.jdbc("jdbc:mysql://20.0.20.203/testdb", "student_infos", connectionProperties)
      val student_scores = spark.read.jdbc("jdbc:mysql://20.0.20.203/testdb", "student_scores", connectionProperties) 
      val studentsRDD = student_infos.join(student_scores,"name").filter("score >=80").rdd
      val structType = StructType(Array(
        StructField("name", StringType, true),
        StructField("score", IntegerType, true),
        StructField("age", IntegerType, true)))
     val studentsDF = spark.createDataFrame(studentsRDD, structType)
     studentsDF.write.jdbc("jdbc:mysql://20.0.20.203/testdb", "good_student_infos", connectionProperties)
      
     spark.close()
 
  }
}