spark streaming scala 过滤黑名单

最新推荐文章于 2022-06-21 13:57:29 发布

贾红平

最新推荐文章于 2022-06-21 13:57:29 发布

阅读量459

点赞数

分类专栏： Spark Scala Streaming系列文章标签： spark streaming

本文链接：https://blog.csdn.net/qq_18603599/article/details/79970278

版权

Spark Scala Streaming系列专栏收录该内容

4 篇文章 0 订阅

订阅专栏

本文章主要通过spark streaming实时过滤黑名单

import org.apache.spark.SparkConf
import org.apache.spark.streaming.StreamingContext
import org.apache.spark.streaming.Seconds

/**
 * @author jhp
  *         spark streaming实时过滤黑名单
 */
object TransformBlacklist {
  
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf()
        .setMaster("local[2]")  
        .setAppName("TransformBlacklist")
    //实例化流上下文
    val ssc = new StreamingContext(conf, Seconds(5))
    
    val blacklist = Array(("tom", true))  
    val blacklistRDD = ssc.sparkContext.parallelize(blacklist, 5)  
    
    val adsClickLogDStream = ssc.socketTextStream("spark1", 9999)   
    val userAdsClickLogDStream = adsClickLogDStream
        .map { adsClickLog => (adsClickLog.split(" ")(1), adsClickLog) } 
    //转换RDD
    val validAdsClickLogDStream = userAdsClickLogDStream.transform(userAdsClickLogRDD => {
      val joinedRDD = userAdsClickLogRDD.leftOuterJoin(blacklistRDD)
      //RDD过滤
      val filteredRDD = joinedRDD.filter(tuple => {
        if(tuple._2._2.getOrElse(false)) {  
          false
        } else {
          true
        }
      })
      val validAdsClickLogRDD = filteredRDD.map(tuple => tuple._2._1) 
      validAdsClickLogRDD
    })
    
    validAdsClickLogDStream.print()
    //启动上下文
    ssc.start()
    ssc.awaitTermination()
  }
  
}