使用json文件创建 dataframe演示常用操作
java版本:
object dataframecreate {
def main(args:Array[String]){
val conf = new SparkConf()
.setAppName("dataframecreate")
val sc = new SparkContext(conf)
val sqlcontext = new SQLContext(sc)
val df = sqlcontext.read.json("hdfs://master:9000/student.json")
df.show()
df.printSchema()
df.select(df.col("name"),df.col("age").plus(1)).show()
df.select(df.col("name"), df.col("age") + 1).show()
df.filter(df.col("age").gt(20)).show()
df.groupBy("name").count().show()
}
}
java版本:
package cn.spark.study.core;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.DataFrame;
import org.apache.spark.sql.SQLContext;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.DataFrame;
import org.apache.spark.sql.SQLContext;
public class dataframecreate {
public static void main(String[] args) {
SparkConf conf = new SparkConf()
.setAppName("dataframecreate");
JavaSparkContext sc = new JavaSparkContext(conf);
SQLContext sqlContext = new SQLContext(sc);
DataFrame df = sqlContext.read().json("hdfs://master:9000/student.json");
df.show();
df.printSchema();
df.select("name").show();
df.select(df.col("name")).show();
df.select(df.col("name"),df.col("age").plus(3)).show();
df.filter(df.col("age").gt(20)).show();
df.groupBy(df.col("name")).count().show();
}
}
scala版本:
public static void main(String[] args) {
SparkConf conf = new SparkConf()
.setAppName("dataframecreate");
JavaSparkContext sc = new JavaSparkContext(conf);
SQLContext sqlContext = new SQLContext(sc);
DataFrame df = sqlContext.read().json("hdfs://master:9000/student.json");
df.show();
df.printSchema();
df.select("name").show();
df.select(df.col("name")).show();
df.select(df.col("name"),df.col("age").plus(3)).show();
df.filter(df.col("age").gt(20)).show();
df.groupBy(df.col("name")).count().show();
}
}
package com.spark.study.core
import org.apache.spark.SparkConf
import org.apache.spark.SparkContext
import org.apache.spark.sql.SQLContext
import org.apache.spark.SparkContext
import org.apache.spark.sql.SQLContext
object dataframecreate {
def main(args:Array[String]){
val conf = new SparkConf()
.setAppName("dataframecreate")
val sc = new SparkContext(conf)
val sqlcontext = new SQLContext(sc)
val df = sqlcontext.read.json("hdfs://master:9000/student.json")
df.show()
df.printSchema()
df.select(df.col("name"),df.col("age").plus(1)).show()
df.select(df.col("name"), df.col("age") + 1).show()
df.filter(df.col("age").gt(20)).show()
df.groupBy("name").count().show()
}
}
来自 “ ITPUB博客 ” ,链接:http://blog.itpub.net/30541278/viewspace-2154723/,如需转载,请注明出处,否则将追究法律责任。
转载于:http://blog.itpub.net/30541278/viewspace-2154723/