StructredStreaming读取HDFS文件夹示例

package com.itcast.sql

import org.apache.spark.sql.types.{DataType, DoubleType, IntegerType, StringType, StructField, StructType}
import org.apache.spark.sql.{DataFrame, SparkSession}

object StructredStreaming {

  def main(args: Array[String]): Unit = {
    val spark = SparkSession.builder()
      .appName("structrued streaming")
      .master("local[6]")
      .getOrCreate()

    val schema =  StructType(
      Array(StructField("id", IntegerType),
        StructField("name", StringType),
        StructField("course_count", IntegerType),
        StructField("score", DoubleType))
    )

    //setLogLevel
    spark.sparkContext.setLogLevel("error")

    val readDF: DataFrame = spark.readStream
      .schema(schema)
      .csv("hdfs://hadoop100:8020/data/")

    readDF.writeStream
      .format("console")
      .start()
      .awaitTermination()

    spark.stop()
  }


}