import org.apache.spark.streaming.dstream.ReceiverInputDStream
import org.apache.spark.streaming.{Seconds, StreamingContext}
import org.apache.spark.{SparkConf, SparkContext}
/**
* 需要用到nc -lk 9999
* */
/**
* Created by Administrator on 2018/5/11.
*/
object SparkStreaming1 {
def main(args: Array[String]): Unit = {
val conf: SparkConf = new SparkConf().setAppName("workcount").setMaster("local[2]")
//因为只有一个线程
//无法做到一边接收数据一边做transiformation处理
val sc = new SparkContext(conf)
val scc = new StreamingContext(sc,Seconds(2))
//创建StreamingContext程序入口
val dstream: ReceiverInputDStream[String] = scc.socketTextStream("hadoop02",9999)
//监听虚拟机端口发送的消息
/**
* 进行单词计数
* */
dstream.
flatMap(line=>line.split(" ")).
map(x=>(x,1)).
reduceByKey(_+_).print()
/**
* print操作类似collect
* */
scc.start()
scc.awaitTermination()
scc.stop()
/**
* 后面三个固定搭配
* */
}
}
SparkStreaming的单词计数案例
最新推荐文章于 2024-05-28 09:19:48 发布