Spark2.X Streaming+kafka wordCount代码

最新推荐文章于 2022-06-02 16:10:29 发布

坚持的道路注定孤独

最新推荐文章于 2022-06-02 16:10:29 发布

阅读量314

点赞数

分类专栏： Spark2.0

本文链接：https://blog.csdn.net/qq_31136839/article/details/79850570

版权

Spark2.0 专栏收录该内容

4 篇文章 0 订阅

订阅专栏

package cn.spark.study.streaming;

import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;

import org.apache.spark.SparkConf;
import org.apache.spark.streaming.Durations;
import org.apache.spark.streaming.api.java.JavaDStream;
import org.apache.spark.streaming.api.java.JavaPairDStream;
import org.apache.spark.streaming.api.java.JavaPairReceiverInputDStream;
import org.apache.spark.streaming.api.java.JavaStreamingContext;
import org.apache.spark.streaming.api.java.JavaReceiverInputDStream;
import org.apache.spark.streaming.kafka010.KafkaUtils;

import scala.Tuple2;
public class KafkaReceiverWordCount {

	public static void main(String[] args) {
		
		SparkConf conf = new SparkConf()
								.setMaster("local[4]")
								.setAppName("KafkaReceiverWordcount");
		JavaStreamingContext jssc = new JavaStreamingContext(conf,Durations.seconds(5));
		
		Map<String, Integer> topicThreaMap = new HashMap<String,Integer>();
		
		topicThreaMap.put("WordCount", 1);
		
		
		JavaPairReceiverInputDStream<String, String> lines = KafkaUtils.createStream(
				jssc, 
				"20.0.20.201:2181,20.0.20.202:2181,20.0.20.203:2181", 
				"DefaultConsumerGroup", 
				topicThreaMap);
		JavaDStream<String> words = lines.flatMap(x -> Arrays.asList(x._2.split(" ")).iterator());
		
		JavaPairDStream<String, Integer> pairs =words.mapToPair(x -> new Tuple2(x,1));
		
		JavaPairDStream<String, Integer> wordCount = pairs.reduceByKey((i1,i2) -> i1+ i2);
	
		wordCount.print();
		
		jssc.start();
		jssc.awaitTermination();
		jssc.close();
	}
}