package com.educoder.bigData.sparksql;import org.apache.spark.sql.AnalysisException;import org.apache.spark.sql.SparkSession;publicclassTest2{publicstaticvoidmain(String[] args) throws AnalysisException {
SparkSession spark = SparkSession
.builder().appName("test1").master("local").config("spark.some.config.option","some-value").getOrCreate();/********* Begin *********///读取json,并将Dataset,并注册为SQL临时视图
spark.read().json("people.json").createOrReplaceTempView("people");
spark.sql("select * from people where age !='23'").show();/********* End *********/}}
第3关 Dataset自定义函数
package com.educoder.bigData.sparksql;import java.util.ArrayList;import java.util.List;import org.apache.spark.sql.Row;import org.apache.spark.sql.expressions.MutableAggregationBuffer;import org.apache.spark.sql.expressions.UserDefinedAggregateFunction;import org.apache.spark.sql.types.DataType;import org.apache.spark.sql.types.DataTypes;import org.apache.spark.sql.types.StructField;import org.apache.spark.sql.types.StructType;publicclassMyAverageextendsUserDefinedAggregateFunction{privatestatic final long serialVersionUID = 1L;private StructType inputSchema;private StructType bufferSchema;publicMyAverage(){/********* Begin *********/
List<StructField> inputFields =newArrayList<StructField>();
inputFields.add(DataTypes.createStructField("inputColumn", DataTypes. LongType,true));
inputSchema = DataTypes.createStructType(inputFields);
List<StructField> bufferFields =newArrayList<StructField>();
bufferFields.add(DataTypes.createStructField("sum", DataTypes.LongType,true));
bufferFields.add(DataTypes.createStructField("count", DataTypes.LongType,true));
bufferSchema = DataTypes.createStructType(bufferFields);/********* End *********/}
@Override
public StructType bufferSchema(){/********* Begin *********///return null;return bufferSchema;/********* End *********/}
@Override
public DataType dataType(){/********* Begin *********///return null;return DataTypes.DoubleType;/********* End *********/}
@Override
public boolean deterministic(){// TODO Auto-generated method stubreturntrue;}
@Override
public Object evaluate(Row buffer){/********* Begin *********///return null;return((double) buffer.getLong(0))/ buffer.getLong(1);/********* End *********/}
@Override
publicvoidinitialize(MutableAggregationBuffer buffer){/********* Begin *********/
buffer.update(0, 0L);
buffer.update(1, 0L);/********* End *********/}
@Override
public StructType inputSchema(){/********* Begin *********///return null;return inputSchema;/********* End *********/}
@Override
publicvoidmerge(MutableAggregationBuffer buffer1, Row buffer2){/********* Begin *********/
long mergedSum = buffer1.getLong(0)+ buffer2.getLong(0);
long mergedCount = buffer1.getLong(1)+ buffer2.getLong(1);
buffer1.update(0, mergedSum);
buffer1.update(1, mergedCount);/********* End *********/}
@Override
publicvoidupdate(MutableAggregationBuffer buffer, Row input){/********* Begin *********/if(!input.isNullAt(0)){
long updatedSum = buffer.getLong(0)+ input.getLong(0);
long updatedCount = buffer.getLong(1)+1;
buffer.update(0, updatedSum);
buffer.update(1, updatedCount);}/********* End *********/}}
第1关 SparkSQL初识package com.educoder.bigData.sparksql; import org.apache.spark.sql.AnalysisException;import org.apache.spark.sql.Dataset;import org.apache.spark.sql.Row;import org.apache.spark.sql.SparkSession; public class Test1 { public stat