import org.apache.spark.rdd.RDD
import org.apache.spark.streaming.dstream.ReceiverInputDStream
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.streaming.{Seconds, StreamingContext}
object TransformDemo {
def main(args: Array[String]): Unit = {
val conf =new SparkConf().setMaster("local[2]").setAppName(this.getClass.getSimpleName)
val ssc: StreamingContext = new StreamingContext(conf,Seconds(3))
val testUser: RDD[(String, Boolean)] = ssc.sparkContext.makeRDD(Array(("tom",true),("jim",true)))
val input: ReceiverInputDStream[String] = ssc.socketTextStream("b02master",1214)
input.transform(rdd=>{
val allUser: RDD[(String, String)] = rdd.map(line=> ((line.split(" +")(1),line)))
val joinRdd: RDD[(String, (String, Option[Boolean]))] = allUser.leftOuterJoin(testUser)
val filterRdd = joinRdd.filter(tp => {
if (tp._2._2.getOrElse(false)) false
else true
})
filterRdd.map(_._2._1)
}).print()
ssc.start()
ssc.awaitTermination()
}
}
TransformDemo
最新推荐文章于 2020-07-23 07:47:29 发布