import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.sql.SparkSession
object SparkDataSet {
def main(args: Array[String]): Unit = {
val conf = new SparkConf().setMaster("local[2]").setAppName("test").set("spark.testing.memory","520000000")
val spark = SparkSession.builder().config(conf).getOrCreate()
val sc = SparkContext.getOrCreate()
case class Person(name:String,age:Long)
import spark.implicits._
val caseDf = Seq(Person("jack",11)).toDF()
caseDf.show()
}
}
Error:(57, 131) value toDF is not a member of org.apache.spark.rdd.RDD[Person]
val personDf = spark.sparkContext.textFile(path).map(_.split(",")).map(attributes=>Person(attributes(0),attributes(1).toInt)).toDF()`
person 是我自定义的一个样例类
经检查百度发现是因为我把定义的样例类放在了main方法内,跟要执行的创建的dataFrame语句在一个方法内了