package com.bjsxt;
import java.util.Arrays;
import java.util.List;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.api.java.function.PairFunction;
import org.apache.spark.broadcast.Broadcast;
import org.apache.spark.streaming.Durations;
import org.apache.spark.streaming.api.java.JavaDStream;
import org.apache.spark.streaming.api.java.JavaPairDStream;
import org.apache.spark.streaming.api.java.JavaReceiverInputDStream;
import org.apache.spark.streaming.api.java.JavaStreamingContext;
import groovy.lang.Tuple;
import scala.Tuple2;
public class TransformOperator {
public static void main(String[] args) {
SparkConf conf = new SparkConf().setAppName("test").setMaster("local");
/**
* 设置JavaStreamingContext
*/
JavaStreamingContext jsc = new JavaStreamingContext(conf, Durations.seconds(5));
/**
* 设定list的初始值为zhangsan
*/
List<String> list = Arrays.asList("zhangsan");
/**
* bcBlackList必须要用final修饰
*/
final Broadcast<List<String>> bcBlackList = jsc.sparkContext().broadcast(list);
/**
* 接受来自Linux端你的数据
*/
JavaReceiverInputDStream<String> nameList = jsc.socketTextStream("node01", 9999);
/**
* 调用maptopair:
* 进行切分
* 进行输出
*/
JavaPairDStream<String, String> pairnamelist = nameList.mapToPair(new PairFunction<String, String, String>() {
@Override
public Tuple2<String, String> call(String line) throws Exception {
return new Tuple2<String, String>(line.split(" ")[1], line);
}
});
/**
*
*
*/
/**
* 调用transform算子
*/
JavaDStream<String> transformresult = pairnamelist.transform(new Function<JavaPairRDD<String, String>, JavaRDD<String>>() {
@Override
public JavaRDD<String> call(JavaPairRDD<String, String> nameRDD) throws Exception {
/**
* 自己创建nameRDD
* 调用filter进行过滤
*/
JavaPairRDD<String, String> filter = nameRDD.filter(new Function<Tuple2<String, String>, Boolean>() {
@Override
public Boolean call(Tuple2<String, String> Tuple) throws Exception {
/**
* 过滤:将符合条件的留下来
* 在这里是根据key去过滤的
*/
return !bcBlackList.value().contains(Tuple._1);
}
});
/**
* 调用map算子,传进去的是tuple格式的,传出来的是tuple的值
*/
JavaRDD<String> map = filter.map(new Function<Tuple2<String, String>, String>() {
@Override
public String call(Tuple2<String, String> Tuple) throws Exception {
return Tuple._2;
}
});
/**
* 最终返回map
*/
// 返回过滤好的结果
return map;
}
});
/**
* transformresult
* 这里边的内容是最后返回的map中的东西
* 是tuple的值
*/
transformresult.print();
jsc.start();
jsc.awaitTermination();
jsc.stop();
}
}
SparkStreaming部分:transform算子(在transform内部将传进来不符合的内容按照key过滤掉,下剩的东西按照map算子,以tuple值的形式输出)【Java版纯代码】
最新推荐文章于 2022-08-17 18:21:47 发布