spark系列二：sparksql dataframe常用操作

使用json文件创建 dataframe演示常用操作
java版本：

package cn.spark.study.core;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.DataFrame;
import org.apache.spark.sql.SQLContext;

public class dataframecreate {
  public static void main(String[] args) {
 SparkConf conf = new SparkConf()
   .setAppName("dataframecreate");
 JavaSparkContext sc = new JavaSparkContext(conf);
 SQLContext sqlContext = new SQLContext(sc);
 
 
 
 DataFrame df = sqlContext.read().json("hdfs://master:9000/student.json");
 
 
 df.show();
 df.printSchema();
 df.select("name").show();
 df.select(df.col("name")).show();
 df.select(df.col("name"),df.col("age").plus(3)).show();
 df.filter(df.col("age").gt(20)).show();
 df.groupBy(df.col("name")).count().show(); 
}
}

scala版本：

package com.spark.study.core

import org.apache.spark.SparkConf
import org.apache.spark.SparkContext
import org.apache.spark.sql.SQLContext

object dataframecreate {
  def main(args:Array[String]){
    val conf = new SparkConf()
                  .setAppName("dataframecreate")
                
    val sc = new SparkContext(conf)
    val sqlcontext = new SQLContext(sc)
    val df = sqlcontext.read.json("hdfs://master:9000/student.json")
    df.show()
    df.printSchema()
    df.select(df.col("name"),df.col("age").plus(1)).show()
    df.select(df.col("name"), df.col("age") + 1).show()
    df.filter(df.col("age").gt(20)).show()
    df.groupBy("name").count().show()
   
   
  }
}

来自 “ ITPUB博客 ” ，链接：http://blog.itpub.net/30541278/viewspace-2154723/，如需转载，请注明出处，否则将追究法律责任。

转载于:http://blog.itpub.net/30541278/viewspace-2154723/