Spark-SQL 读写Orc 文件

• 读文件

import org.apache.spark.sql.{DataFrame, SparkSession}

//通过csv文件创建DataFrame
object CreateDataFrameFromOrc {

  def main(args: Array[String]): Unit = {

    //创建SparkSession(是对SparkContext的包装和增强)
    val spark: SparkSession = SparkSession.builder()
      .appName(this.getClass.getSimpleName)
      .master("local[*]")
      .getOrCreate()


    val df: DataFrame = spark.read.orc("/Users/Desktop/orc")

    df.show(2)

    df.printSchema()


  }
}

• 写文件

import org.apache.spark.rdd.RDD
import org.apache.spark.sql.types._
import org.apache.spark.sql.{Row, SparkSession}

object WriteToOrc {

  def main(args: Array[String]): Unit = {

    //创建SparkSession(是对SparkContext的包装和增强)
    val session: SparkSession = SparkSession.builder()
      .appName("DataFrameCreateDemo01")
      .master("local[*]")
      .getOrCreate()

    val sc = session.sparkContext
    val rdd1: RDD[String] = sc.textFile("hdfs://node-1.cn:9000/userdata")

    //row的字段没有类型，没有名称
    val rowRDD: RDD[Row] = rdd1.map(e => {
      val fields = e.split(",")
      Row(fields(0).toLong, fields(1), fields(2).toInt, fields(3).toDouble)
    })

    //关联schema(字段名称，数据类型，是否可以为空)
    val schema = StructType(
      Array(
        StructField("id", LongType),
        StructField("name", StringType),
        StructField("age", IntegerType),
        StructField("fv", DoubleType),
      )
    )
    //将RowRDD和StructType类型的Schema进行关联
    val df = session.createDataFrame(rowRDD, schema)

    df.write.orc("/Users/Desktop/orc")

    session.stop()

  }
}