说明:
屏蔽黑名单…
package com.qf.sparkStreaming.day02
import org.apache.log4j.{Level, Logger}
import org.apache.spark.SparkConf
import org.apache.spark.rdd.RDD
import org.apache.spark.streaming.dstream.{DStream, ReceiverInputDStream}
import org.apache.spark.streaming.{Seconds, StreamingContext}
object _03Blacklist {
def main(args: Array[String]): Unit = {
//只保留warn以上的信息
Logger.getLogger("org").setLevel(Level.WARN) //记得导org.apache.log4j的包
//1.获取配置对象
val conf: SparkConf = new SparkConf().setAppName("test").setMaster("local[2]")
//2.获取上下文对象
val ssc: StreamingContext = new StreamingContext(conf, Seconds(10))
//3.读取数据流
val dStream: ReceiverInputDStream[String] = ssc.socketTextStream("qianfeng01", 10086)
//模拟一个rdd来存储黑名单
val list: List[(String, Boolean)] = List(("mei", true), ("ya", true), ("zhao", true), ("ju", true))
val blackList: RDD[(String, Boolean)] = ssc.sparkContext.makeRDD(list)
//4.让micro-batch里的数据与黑名单进行关联【左外连接】,1是随便写的,没有影响
// 比如:小白 (1,none)
// mei (1,true) <=不要
val dStream1: DStream[(String, (Int, Option[Boolean]))] = dStream.map((_, 1)).transform((rdd, time) => {
val rdd1: RDD[(String, (Int, Option[Boolean]))] = rdd.leftOuterJoin(blackList)
val rdd2: RDD[(String, (Int, Option[Boolean]))] = rdd1.filter(x => {
if (x._2._2.getOrElse(false)) {
false
} else {
true
}
})
rdd2
})
//5.打印
dStream1.print()
//6.启动
ssc.start()
ssc.awaitTermination()
}
}
执行:
nc -lk qianfeng01 10086