6.SparkStreaming-屏蔽黑名单

说明：

屏蔽黑名单…

package com.qf.sparkStreaming.day02

import org.apache.log4j.{Level, Logger}
import org.apache.spark.SparkConf
import org.apache.spark.rdd.RDD
import org.apache.spark.streaming.dstream.{DStream, ReceiverInputDStream}
import org.apache.spark.streaming.{Seconds, StreamingContext}

object _03Blacklist {



  def main(args: Array[String]): Unit = {

    //只保留warn以上的信息
    Logger.getLogger("org").setLevel(Level.WARN) //记得导org.apache.log4j的包

    //1.获取配置对象
    val conf: SparkConf = new SparkConf().setAppName("test").setMaster("local[2]")
    //2.获取上下文对象
    val ssc: StreamingContext = new StreamingContext(conf, Seconds(10))
    //3.读取数据流
    val dStream: ReceiverInputDStream[String] = ssc.socketTextStream("qianfeng01", 10086)
    //模拟一个rdd来存储黑名单
    val list: List[(String, Boolean)] = List(("mei", true), ("ya", true), ("zhao", true), ("ju", true))
    val blackList: RDD[(String, Boolean)] = ssc.sparkContext.makeRDD(list)

    //4.让micro-batch里的数据与黑名单进行关联【左外连接】，1是随便写的，没有影响
    // 比如：小白 (1,none)
    // mei (1,true) <=不要
    val dStream1: DStream[(String, (Int, Option[Boolean]))] = dStream.map((_, 1)).transform((rdd, time) => {
      val rdd1: RDD[(String, (Int, Option[Boolean]))] = rdd.leftOuterJoin(blackList)
      val rdd2: RDD[(String, (Int, Option[Boolean]))] = rdd1.filter(x => {
        if (x._2._2.getOrElse(false)) {
          false
        } else {
          true
        }
      })
      rdd2
    })
    //5.打印
    dStream1.print()

    //6.启动
    ssc.start()
    ssc.awaitTermination()
  }

}

执行：

nc -lk qianfeng01 10086