spark小记——scala的Map类型转sparksql的dataframe

源码：

package com.ydj.sql

import org.apache.spark.sql.SparkSession

import scala.collection.mutable.ArrayBuffer

/**
  * @Auther: yuandejin
  * @Date: 2019-09-04 13:43
  * @Description: 
  */
object MapToDataFrame {


  def main(args: Array[String]): Unit = {

    //1. 构建sparkSession
    val sparkSession = SparkSession
      .builder()
      .appName("MapToDataFrame")
      .master("local")
      .getOrCreate()

    //2. 隐式转换 rdd转dataFrame
    import sparkSession.implicits._
    
    //3. scala的Map数据结构
    val map = Map("aa" -> "aaa", "bb" -> "bbb")

    //4. map的所有key
    val mk = map.keys
    
    //5. 创建rdd
    val rdd = sparkSession.sparkContext.parallelize(Seq(map))
    
    //6. 根据map的key取出所有的值，构建新的rdd，并转成dataFrame 
    val frame = rdd.map(x => {
        val bb = new ArrayBuffer[String]()
        for (k: String <- mk) {
          bb.+=(x(k))
        }
        bb
      }).map(x => (x(0), x(1))).toDF("k1", "k2")
    //7. 打印
    frame.show()
  }
}

运行结果：