package com.zla.test;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.api.java.function.MapFunction;
import org.apache.spark.api.java.function.VoidFunction;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SparkSession;
public class test {
public static void main(String[] args) {
SparkConf sparkConf = new SparkConf();
sparkConf.setAppName("Simple Application")
.set("spark.testing.memory",(512 * 1014 * 1024)+"")
.setMaster("local[*]");
SparkSession session = SparkSession.builder().config(sparkConf).getOrCreate();
Dataset<String> stringDataset = session.read().textFile("D:\\workspace\\sparkdemo\\src\\main\\resources\\person.txt");
JavaRDD<String> stringJavaRDD = stringDataset.toJavaRDD();
JavaRDD<Person> personRdd = stringJavaRDD.map(new Function<String, Person>() {
@Override
public Person call(String s) throws Exception {
return new Person(s.split(" ")[0], Integer.parseInt(s.split(" ")[1]));
}
});
personRdd.foreach(new VoidFunction<Person>() {
@Override
public void call(Person person) throws Exception {
System.out.println(person.toString());
}
});
Dataset<Row> dataFrame = session.createDataFrame(personRdd, Person.class);
dataFrame.show();
}
}
上面的Person类一定要写上get和set方法,否则无法获取Person的类型
package com.zla.test;
public class Person {
public String name;
public Integer age;
public Person() {
}
public Person(String name, Integer age) {
this.name = name;
this.age = age;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public Integer getAge() {
return age;
}
public void setAge(Integer age) {
this.age = age;
}
@Override
public String toString() {
return "Person{" +
"name='" + name + '\'' +
", age=" + age +
'}';
}
}