map
String转为Tuple2<String, Integer>
public class test {
public static void main(String[] args) {
SparkConf conf = new SparkConf().setAppName("jiangtao_demo").setMaster("local");
JavaSparkContext jsc = new JavaSparkContext(conf);
//并行集合生成JavaRDD
JavaRDD lines = jsc.parallelize(Arrays.asList("pandas pip", "numpy", "pip", "pip", "pip"));
//map 输出包含newTail的元素
//这个是给每个元素尾部加newTail
JavaRDD<String> mapResult = lines.map(new Function<String, String>() {
@Override
public String call(String o) throws Exception {
return o.concat("newTail");
}
});
//下面这个是给每个元素String转为Tuple2<String, Integer>元组类型的
//map算子输入与输出一对一,输出结果为
// [(pandas, 1),(pip, 1)]
// [(numpy, 1)]
// [(pip, 1)]
// [(pip, 1)]
// [(pip, 1)]
JavaRDD<Iterable<Tuple2<String, Integer>>> maprdd = lines.map(new Function<String, Iterable<Tuple2<String, Integer>>>() {
public Iterable<Tuple2<String, Integer>> call(String line) throws Exception {
String[] fields = line.split(" ");
ArrayList<Tuple2<String, Integer>> al = new ArrayList<Tuple2<String, Integer>>();
for (int i = 0; i < fields.length; i++) {
al.add(new Tuple2(fields[i], 1));
}
return al;
}
});
maprdd.foreach(s -> {
System.out.println(s);
});
}
}
flatMap
<String>类型的转为<String>
这里比如一行元素有两个单词,使用这个方法可以将这一行元素拆为两行元素,每行元素一个单词
public class test {
public static void main(String[] args) {
SparkConf conf = new SparkConf().setAppName("jiangtao_demo").setMaster("local");
JavaSparkContext jsc = new JavaSparkContext(conf);
JavaRDD lines = jsc.parallelize(Arrays.asList("pandas pip", "numpy", "pip", "pip", "pip"));
// flatMap
// 与输入是一对多的关系,返回的是可迭代的List
// pandas
// pip
// numpy
// pip
// pip
// pip
JavaRDD<String> flatMapResult = lines.flatMap(new FlatMapFunction<String, String>() {
@Override
public Iterator<String> call(String line) throws Exception {
return Arrays.asList(line.toString().split(" ")).iterator();
}
});
flatMapResult.foreach(s -> {
System.out.println(s);
});
}
}
flatMapToPair
<String>类型的转为<String, Integer>
将每一行的元素拆分后再转为键值对类型的RDD
public class test {
public static void main(String[] args) {
SparkConf conf = new SparkConf().setAppName("jiangtao_demo").setMaster("local");
JavaSparkContext jsc = new JavaSparkContext(conf);
JavaRDD lines = jsc.parallelize(Arrays.asList("pandas pip", "numpy", "pip", "pip", "pip"));
// flatMapToPair
// 注意返回值,返回的是list,里面封装了tuple2
// (pandas,1)
// (pip,1)
// (numpy,1)
// (pip,1)
// (pip,1)
// (pip,1)
JavaPairRDD<String, Integer> result1 = lines.flatMapToPair(new PairFlatMapFunction<String, String, Integer>() {
@Override
public Iterator<Tuple2<String, Integer>> call(String line) throws Exception {
String[] fields = line.split(" ");
ArrayList<Tuple2<String, Integer>> al = new ArrayList<Tuple2<String, Integer>>();
for (int i = 0; i < fields.length; i++) {
al.add(new Tuple2<String, Integer>(fields[i], 1));
}
return al.iterator();
}
});
result1.foreach(s -> {
System.out.println(s);
});
}
}