SparkStreaming 读取Hdfs

最新推荐文章于 2021-01-14 01:04:35 发布

weixin_34405332

最新推荐文章于 2021-01-14 01:04:35 发布

阅读量397

点赞数

文章标签：大数据 scala java

原文链接：https://my.oschina.net/momisabuilder/blog/720112

版权

2019独角兽企业重金招聘Python工程师标准>>>

一、代码

SparkConf conf = new SparkConf().setAppName("spark streaming tst").setMaster("local");

JavaStreamingContext javaStreamingContext = new JavaStreamingContext(conf, Durations.seconds(60));


//TODO 切记这是目录 目录 目录 然后动态的往里面加文件
JavaDStream<String> wordRDD = javaStreamingContext.textFileStream("/lwj/second/");


JavaPairDStream<String, Integer> wordsRDD = wordRDD.mapToPair(new PairFunction<String, String, Integer>() {
            @Override
            public Tuple2<String, Integer> call(String s) throws Exception {

                return new Tuple2<String, Integer>(s, 1);

            }
        }).reduceByKey(new Function2<Integer, Integer, Integer>() {
            @Override
            public Integer call(Integer v1, Integer v2) throws Exception {
                return v1 + v2;
            }
        }).persist(StorageLevel.MEMORY_ONLY());


        wordsRDD.print();

        javaStreamingContext.start();

        javaStreamingContext.awaitTermination();