2021-10-20

Exception in thread “main” org.apache.spark.sql.AnalysisException: cannot resolve ‘age’ given input columns: []; line 1 pos 37;

其实就是 建立的类Student中没有为相应的属性增加get和set方法，增加相应方法后问题解决

代码：
package com.imooc.java.sql;

import java.io.Serializable;

public class Student implements Serializable {
private String name ;
private Integer age;
//
// public String getName() {
// return name;
// }
//
// public void setName(String name) {
// this.name = name;
// }
//
// public Integer getAge() {
// return age;
// }
//
// public void setAge(Integer age) {
// this.age = age;
// }

public Student(String name, Integer age) {
    this.name = name;
    this.age = age;
}

@Override
public String toString() {
    return "Student{" +
            "name='" + name + '\'' +
            ", age=" + age +
            '}';
}

}

package com.imooc.java.sql;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SparkSession;
import scala.Tuple2;

import java.util.Arrays;
import java.util.List;

/**

• 需求：使用反射方式实现RDD转换为DataFrame

• Created by xuxinxing
  */
  public class RddToDataFrameByReflectJava {
  public static void main(String[] args) {
  SparkConf conf = new SparkConf();
  conf.setMaster(“local”);
  SparkSession sparkSession = SparkSession.builder()
  .appName(“RddToDataFrameByReflectJava”)
  .config(conf)
  .getOrCreate();
  JavaSparkContext sc = JavaSparkContext.fromSparkContext(sparkSession.sparkContext());
  Tuple2<String, Integer> tup1 = new Tuple2<String, Integer>(“xuxinxing”, 31);
  Tuple2<String, Integer> tup2 = new Tuple2<String, Integer>(“zhangsan”, 17);
  Tuple2<String, Integer> tup3 = new Tuple2<String, Integer>(“王五”, 21);
  JavaRDD<Tuple2<String, Integer>> dataRDD = sc.parallelize(Arrays.asList(tup1, tup2, tup3));
  JavaRDD stuRDD = dataRDD.map(new Function<Tuple2<String, Integer>, Student>() {
  @Override
  public Student call(Tuple2<String, Integer> tup) throws Exception {
  return new Student(tup._1, tup._2);
  }
  });
  Dataset stuDf = sparkSession.createDataFrame(stuRDD , Student.class);
  stuDf.createOrReplaceTempView(“t_student”);
  Dataset resDf = sparkSession.sql(“select name,age from t_student where age >18”);
  JavaRDD resRDD = resDf.javaRDD();

   List<Student> resList = resRDD.map(new Function<Row, Student>() {
       @Override
       public Student call(Row row) throws Exception {
           return new Student(row.getAs("name").toString(), Integer.parseInt(row.getAs("age").toString()));
       }
   }).collect();
   for(Student stu:resList){
       System.out.println(stu);
   }
   sparkSession.stop();
  

  }
  }

报错：Exception in thread “main” org.apache.spark.sql.AnalysisException: cannot resolve ‘age’ given input columns: []; line 1 pos 37;
'Project ['name, 'age]
± 'Filter ('age > 18)
± SubqueryAlias t_student
± LogicalRDD false

at org.apache.spark.sql.catalyst.analysis.package$AnalysisErrorAt.failAnalysis(package.scala:42)
at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1$$anonfun$apply$3.applyOrElse(CheckAnalysis.scala:110)
at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1$$anonfun$apply$3.applyOrElse(CheckAnalysis.scala:107)
at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformUp$1.apply(TreeNode.scala:278)
at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformUp$1.apply(TreeNode.scala:278)
at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(TreeNode.scala:70)
at org.apache.spark.sql.catalyst.trees.TreeNode.transformUp(TreeNode.scala:277)
at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$3.apply(TreeNode.scala:275)
at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$3.apply(TreeNode.scala:275)
at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$4.apply(TreeNode.scala:326)
at org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:187)
at org.apache.spark.sql.catalyst.trees.TreeNode.mapChildren(TreeNode.scala:324)
at org.apache.spark.sql.catalyst.trees.TreeNode.transformUp(TreeNode.scala:275)
at org.apache.spark.sql.catalyst.plans.QueryPlan$$anonfun$transformExpressionsUp$1.apply(QueryPlan.scala:93)
at org.apache.spark.sql.catalyst.plans.QueryPlan$$anonfun$transformExpressionsUp$1.apply(QueryPlan.scala:93)
at org.apache.spark.sql.catalyst.plans.QueryPlan$$anonfun$1.apply(QueryPlan.scala:105)
at org.apache.spark.sql.catalyst.plans.QueryPlan$$anonfun$1.apply(QueryPlan.scala:105)
at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(TreeNode.scala:70)
at org.apache.spark.sql.catalyst.plans.QueryPlan.transformExpression$1(QueryPlan.scala:104)
at org.apache.spark.sql.catalyst.plans.QueryPlan.org$apache$spark$sql$catalyst$plans$QueryPlan$$recursiveTransform$1(QueryPlan.scala:116)
at org.apache.spark.sql.catalyst.plans.QueryPlan$$anonfun$2.apply(QueryPlan.scala:126)
at org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:187)
at org.apache.spark.sql.catalyst.plans.QueryPlan.mapExpressions(QueryPlan.scala:126)
at org.apache.spark.sql.catalyst.plans.QueryPlan.transformExpressionsUp(QueryPlan.scala:93)
at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1.apply(CheckAnalysis.scala:107)
at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1.apply(CheckAnalysis.scala:85)
at org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:127)
at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:126)
at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:126)
at scala.collection.immutable.List.foreach(List.scala:392)
at org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:126)
at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$class.checkAnalysis(CheckAnalysis.scala:85)
at org.apache.spark.sql.catalyst.analysis.Analyzer.checkAnalysis(Analyzer.scala:95)
at org.apache.spark.sql.catalyst.analysis.Analyzer$$anonfun$executeAndCheck$1.apply(Analyzer.scala:108)
at org.apache.spark.sql.catalyst.analysis.Analyzer$$anonfun$executeAndCheck$1.apply(Analyzer.scala:105)
at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper$.markInAnalyzer(AnalysisHelper.scala:201)
at org.apache.spark.sql.catalyst.analysis.Analyzer.executeAndCheck(Analyzer.scala:105)
at org.apache.spark.sql.execution.QueryExecution.analyzed$lzycompute(QueryExecution.scala:57)
at org.apache.spark.sql.execution.QueryExecution.analyzed(QueryExecution.scala:55)
at org.apache.spark.sql.execution.QueryExecution.assertAnalyzed(QueryExecution.scala:47)
at org.apache.spark.sql.Dataset$.ofRows(Dataset.scala:78)
at org.apache.spark.sql.SparkSession.sql(SparkSession.scala:642)