1.reduceByKey,只执行当前输入
package com.zpark.stu.sparkstream
import org.apache.log4j.{Level, Logger}
import org.apache.spark.SparkConf
import org.apache.spark.streaming.{Seconds, StreamingContext}
object SparkStreamDemo {
Logger.getLogger("org").setLevel(Level.WARN)
def main(args: Array[String]): Unit = {
val conf = new SparkConf().setAppName("SparkStreamDemo").setMaster("local[*]")
//StreamingContext第二个参数是指定间隔多久执行一次
val ssc = new StreamingContext(conf, Seconds(3))
//hostname指定读取文件的机器,port指定读取文件的端口号
val lines = ssc.socketTextStream("hdp-1", 9999)
lines.flatMap(_.split(" ")).map(x => (x, 1)).reduceByKey(_+_).print()
//启动服务
ssc.start()
//等待结束
ssc.awaitTermination()
}
}
2.updateStateByKey,记录历史记录,其参数需要传入一个更新方法
package com.zpark.stu.sparkstream
import org.apache.log4j.{Level, Logger}
import org.apache.spark.{HashPartitioner, SparkConf}
import org.apache.spark.streaming.{Seconds, StreamingContext}
object SparkStreamDemo2 {
Logger.getLogger("org").setLevel(Level.WARN)
val updateFunc = (iter: Iterator[(String, Seq[Int], Option[Int])]) => {
iter.flatMap{case(x,y,z) => Some(y.sum + z.getOrElse(0)).map(m => (x,m))}
}
def main(args: Array[String]): Unit = {
val conf = new SparkConf().setAppName("SparkStreamDemo2").setMaster("local[*]")
val ssc = new StreamingContext(conf, Seconds(6))
ssc.checkpoint("E:\\hadoop\\spark\\WordCount\\sparkStreamOut")
val lines = ssc.socketTextStream("hdp-1", 9999)
lines.flatMap(_.split(" ")).map((_,1)).updateStateByKey(updateFunc, new HashPartitioner(ssc.sparkContext.defaultParallelism), true).print()
ssc.start()
ssc.awaitTermination()
}
}
3.reduceByKeyAndWindow:设置窗口的大小和滑动窗口的间隔来动态的获取当前Steaming的允许状态,基于窗口的操作会在一个比 StreamingContext 的批次间隔更长的时间范围内,通过整合多个批次的结果,计算出整个窗口的结果。
package com.zpark.stu.window_operations
import org.apache.log4j.{Level, Logger}
import org.apache.spark.SparkConf
import org.apache.spark.streaming.dstream.DStream
import org.apache.spark.streaming.{Seconds, StreamingContext}
object WordCountWindowDemo {
Logger.getLogger("org").setLevel(Level.WARN)
def main(args: Array[String]): Unit = {
val updateFunc = (values: Seq[Int], state: Option[Int]) => {
val currentCount = values.foldLeft(0)(_ + _)
val previousCount = state.getOrElse(0)
Some(currentCount + previousCount)
}
val conf = new SparkConf().setAppName("WordCountWindowDemo").setMaster("local[*]")
val ssc = new StreamingContext(conf, Seconds(6))
ssc.checkpoint("E:\\hadoop\\spark\\WordCount\\sparkWindowOut")
val lens = ssc.socketTextStream("hdp-1",9999)
val mapDs: DStream[(String, Int)] = lens.flatMap(_.split(" ")).map((_,1))
// mapDs.print()
val updateDs: DStream[(String, Int)] = mapDs.updateStateByKey(updateFunc)
// updateDs.print()
//窗口12秒,滑步6秒。
updateDs.reduceByKeyAndWindow((a: Int, b: Int) => (a + b), Seconds(12), Seconds(6)).print()
// val wordCounts: DStream[(String, Int)] = mapDs.reduceByKeyAndWindow((a: Int, b: Int) => (a + b), Seconds(12), Seconds(6))
//
// wordCounts.print()
ssc.start()
ssc.awaitTermination()
}
}