flink笔记：009-013 wordcount

荣晓

已于 2023-03-15 20:17:04 修改

阅读量58

点赞数

分类专栏： flink java 文章标签： flink java 大数据

于 2023-03-15 19:43:19 首次发布

本文链接：https://blog.csdn.net/weixin_43015677/article/details/129565809

版权

flink 同时被 2 个专栏收录

7 篇文章 0 订阅

订阅专栏

java

2 篇文章 0 订阅

订阅专栏

文章展示了如何使用ApacheFlink的DataStreamApi进行有界和无界流的数据处理，包括从文本文件和Socket数据源读取数据，执行WordCount操作。批处理部分则通过BatchWordCount示例解释了如何对文本文件进行读取、分词、分组和求和。

摘要由CSDN通过智能技术生成

dataStream Api

package com.atguigu.wc;

import org.apache.flink.api.common.typeinfo.Types;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.util.Collector;


public class BoundedStreamWordCount {
    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

        //有界流
       // DataStreamSource<String> lineStreamSource = env.readTextFile(BatchWordCount.class.getClassLoader().getResource("word.txt").getPath());

        //真正的流 nc-lk 7777 

        DataStreamSource<String> lineStreamSource = env.socketTextStream("localhost", 7777);
        SingleOutputStreamOperator<Tuple2<String, Long>> wordAndTuple = lineStreamSource.flatMap((String line, Collector<Tuple2<String, Long>> out) -> {
            for (String word : line.split(" ")) {
                out.collect(Tuple2.of(word, 1L));
            }
        }).returns(Types.TUPLE(Types.STRING, Types.LONG));

        wordAndTuple.keyBy(data->data.f0).sum(1).print();

        env.execute();
    }
}

3）> (hello,1) 3 代表本地哪个线程。任务槽（slot）
6> (word,1)
3> (hello,2)
7> (flink,1)
datasetApi

package com.atguigu.wc;

import org.apache.flink.api.common.typeinfo.Types;
import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.api.java.operators.AggregateOperator;
import org.apache.flink.api.java.operators.DataSource;
import org.apache.flink.api.java.operators.FlatMapOperator;
import org.apache.flink.api.java.operators.UnsortedGrouping;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.util.Collector;


public class BatchWordCount {
    public static void main(String[] args) throws Exception {

        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

        DataSource<String> lineDataSource = env.readTextFile(BatchWordCount.class.getClassLoader().getResource("word.txt").getPath());

        FlatMapOperator<String, Tuple2<String, Long>> wordAndTuple = lineDataSource.flatMap((String line, Collector<Tuple2<String, Long>> out) -> {

            for (String key : line.split(" ")) {
                out.collect(Tuple2.of(key, 1L));
            }
        }).returns(Types.TUPLE(Types.STRING, Types.LONG));

        UnsortedGrouping<Tuple2<String, Long>> wordAndgroup = wordAndTuple.groupBy(0);

        AggregateOperator<Tuple2<String, Long>> sum = wordAndgroup.sum(1);

        sum.print();
    }
}