Spark SQL之RDD转换DataFrame的方法

RDD转换DataFrame之Reflection方法

第一种方式是使用反射的方式，用反射去推倒出来RDD里面的schema。这个方式简单，但是不建议使用，因为在工作当中，使用这种方式是有限制的。

对于以前的版本来说，case class最多支持22个大数据培训字段如果超过了22个字段，我们就必须要自己开发一个类，实现product接口才行。因此这种方式虽然简单，但是不通用；因为生产中的字段是非常非常多的，是不可能只有20来个字段的。

 

//Java
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.api.java.function.VoidFunction;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SparkSession;


import javax.jnlp.PersistenceService;
import javax.xml.crypto.Data;


public class rddtoDFreflectionJava {
    public static void main(String[] args) {
        SparkSession spark = SparkSession
                .builder()
                .appName("program")
                .master("local").config("spark.sql.warehouse.dir", "file:/Users/zhangjingyu/Desktop/Spark架构/spark-warehouse")
                .getOrCreate();
        String Path = "file:/Users/zhangjingyu/Desktop/spark-2.4.0/examples/src/main/resources/people.txt";


        JavaRDD<PersonJava> personRDD = Spark.read().textFile(Path).javaRDD().map(line -> {
            String name = line.split(",")[0];
            Long age = Long.valueOf(line.split(",")[1].trim());
            PersonJava person = new PersonJava();
            person.setName(name);
            person.setAge(age);
            return person;
        });
/**
 *         JavaRDD<PersonJava> personRdd = Spark.read().textFile(Path).javaRDD().map(new Function<String, PersonJava>() {
 *             @Override
 *             public PersonJava call(String line) throws Exception {
 *                 String name = line.split(",")[0];
 *                 Long age = Long.valueOf(line.split(",")[1].trim());
 *                 PersonJava person = new PersonJava();
 *                 person.setName(name);
 *                 person.setAge(age);
 *                 return person;
 *             }
 *         });
 */
        Dataset<Row> personDF = Spark.createDataFrame(personRDD,PersonJava.class);
        personDF.createOrReplaceTempView("test");
        Dataset<Row> ResultDF = Spark.sql("select * from test a where a.age < 30");
        ResultDF.show();




        JavaRDD<PersonJava> ResultRDD = ResultDF.javaRDD().map(line -> {
            PersonJava person = new PersonJava();
            person.setName(line.getAs("name"));
            person.setAge(line.getAs("age"));
            return person;
        });


        for (PersonJava personJava : ResultRDD.collect()) {
            System.out.println(personJava.getName()+":"+personJava.getAge());
        }


/**
 *         JavaRDD<PersonJava> resultRdd = ResultDF.javaRDD().map(new Function<Row, PersonJava>() {
 *             @Override
 *             public PersonJava call(Row row) throws Exception {
 *                 PersonJava person = new PersonJava();
 *                 String name = row.getAs("name");
 *                 long age = row.getAs("age");
 *                 person.setName(name);
 *                 person.setAge(age);
 *                 return person;
 *             }
 *         });
 *         resultRdd.foreach(new VoidFunction<PersonJava>() {
 *             @Override
 *             public void call(PersonJava personJava) throws Exception {
 *                 System.out.println(personJava.getName()+":"+personJava.getAge());
 *             }
 *         });
 */
    }
}


//Scala
object rddtoDFreflectionScala {
  case class Person(name : String , age : Long)


  def main(args: Array[String]): Unit = {
    val spark = CommSparkSessionScala.getSparkSession()
    val path = "file:/Users/zhangjingyu/Desktop/spark-2.4.0/examples/src/main/resources/people.txt"
    import spark.implicits._;
    val personDF = spark.sparkContext.textFile(path).map(row => row.split(",")).map(line => {
      Person(line(0),line(1).trim.toLong)
    }).toDF
    personDF.createOrReplaceTempView("test")
    val resultDF = spark.sql("select * from test a where a.age > 20")
    val resultrdd = resultDF.rdd.map(x =>{
      val name = x.getAs[String]("name")
      val age = x.getAs[Long]("age")
      Person(name,age)
    })


    for (elem <- resultrdd.collect()) {
      System.out.println(elem.name+" : "+ elem.age)
    }
  }
}

RDD转换DataFrame之Programm方式

创建一个DataFrame，使用编程的方式，这个方式用的非常多。通过编程方式指定schema ，对于第一种方式的schema其实定义在了case class里面了。

//Java
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.api.java.function.PairFunction;
import org.apache.spark.api.java.function.VoidFunction;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.RowFactory;
import org.apache.spark.sql.SparkSession;
import org.apache.spark.sql.types.DataTypes;
import org.apache.spark.sql.types.StructField;
import org.apache.spark.sql.types.StructType;
import scala.Tuple2;




import java.util.ArrayList;
import java.util.List;


public class rddtoDFprogrammJava {
    public static void main(String[] args) {


        SparkSession spark = SparkSession
                .builder()
                .appName("program")
                .master("local").config("spark.sql.warehouse.dir", "file:/Users/zhangjingyu/Desktop/Spark架构/spark-warehouse")
                .getOrCreate();
        String Path = "file:/Users/zhangjingyu/Desktop/spark-2.4.0/examples/src/main/resources/people.txt";


        //创建列属性
        List<StructField> fields = new ArrayList<>();
        StructField structField_name = DataTypes.createStructField("name", DataTypes.StringType, true);
        StructField structField_age = DataTypes.createStructField("age", DataTypes.LongType, true);
        fields.add(structField_name);
        fields.add(structField_age);
        StructType scheme = DataTypes.createStructType(fields);


        JavaRDD PersonRdd = spark.read().textFile(Path).javaRDD().map(x -> {
            String[] lines = x.split(",");
            return RowFactory.create(lines[0], Long.valueOf(lines[1].trim()));
        });


        Dataset<Row> PersonDF = spark.createDataFrame(PersonRdd, scheme);
        PersonDF.createOrReplaceTempView("program");
        Dataset<Row> ResultDF = spark.sql("select * from program ");
        ResultDF.show();


        for (Row row : ResultDF.javaRDD().collect()) {
            System.out.println(row);
        }
    }
}


//Scala


import org.apache.spark.sql.Row
import org.apache.spark.sql.types.{LongType, StringType, StructField, StructType}


object rddtoDFprogrammScala {
  def main(args: Array[String]): Unit = {
    val spark = CommSparkSessionScala.getSparkSession()
    val path = "file:/Users/zhangjingyu/Desktop/spark-2.4.0/examples/src/main/resources/people.txt"
    val scheme = StructType(Array(
      StructField("name",StringType,true),
      StructField("age",LongType,true)
    ))
    val rdd = spark.sparkContext.textFile(path).map(line => line.split(",")).map(x => {
      Row(x(0),x(1).trim.toLong)
    })
    val PersonDF = spark.createDataFrame(rdd,scheme)
    PersonDF.createOrReplaceTempView("person")
    val resultDF = spark.sql("select * from person a where a.age < 30")
    for (elem <- resultDF.collect()) {
      System.out.println(elem.get(0)+":"+elem.get(1))
    }
  }
}

原创作者：张景宇