使用IDEA开发Spark SQL

指定列名添加Schema

package SparkSql

import org.apache.spark
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.SparkSession
import java.util.Properties

import org.apache.avro.ipc.specific.Person
import org.apache.spark.sql.{
   DataFrame, SaveMode, SparkSession}
object CreateDFDS {
   

  def main(args: Array[String]): Unit = {
   
    import org.apache.spark
    //创建sparksession
    val spark=  SparkSession.builder().master("local[*]")
      .appName("Spark Sql basic example")
      .getOrCreate()
    //通过sparksession对象获得到sparkcontext
    val sc = spark.sparkContext
    //读取数据 转换rdd
    val data: RDD[String] = sc.textFile("input/tt.txt")
    val lineRDD: RDD[Array[String]] = data.map(_.split(" "))
    val rowRDD= lineRDD.map(line =>(line(0).toInt,line(1),line(2).toInt))

    //隐试转换  （作用：）：为对象添加原本不具有的功能/ 方法
    //scala中隐式转换实现了为RDD添加toDF 和toDS
    import spark.implicits._
    val personDF: DataFrame = rowRDD.toDF()
    personDF.show(10)
    personDF.printSchema()
    sc.stop()
    spark.stop()
  

  }
}

StructType指定Schema-了解

package day0413
import org.apache.spark.SparkContext
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.types.{
   IntegerType, StringType, StructField, StructType}
import org.apache.spark.sql.{
   DataFrame, Row, SparkSession}
//  StructType指定Schema
object CreateDFDS2 {
   
  def main(args: Array[String]): Unit = {
   
    // TODO:  StructType指定Schema
    //1.创建SparkSession
    val spark: SparkSession = SparkSession.builder().master("local[*]").appName("SparkSQL").getOrCreate</