flink 一个简单的wordcount

最新推荐文章于 2024-07-22 10:01:15 发布

编码的三叔

最新推荐文章于 2024-07-22 10:01:15 发布

阅读量423

点赞数

文章标签： flink c# 大数据

本文链接：https://blog.csdn.net/ppwwp/article/details/133256234

版权

本文详细介绍了如何使用ApacheFlink的DataStreamAPI进行单词计数，包括数据源读取、分词、分组和聚合操作的过程。

摘要由CSDN通过智能技术生成

package com.jackray.soullan.flink;

import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.api.java.operators.AggregateOperator;
import org.apache.flink.api.java.operators.DataSource;
import org.apache.flink.api.java.operators.FlatMapOperator;
import org.apache.flink.api.java.operators.UnsortedGrouping;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.util.Collector;

public class WordCount {

    public static void main(String[] args) throws Exception {
        // todo 1、创建集群环境
        ExecutionEnvironment executionEnvironment = ExecutionEnvironment.getExecutionEnvironment();

        // todo 2、读取数据、从文件中读取
        DataSource<String> stringDataSource = executionEnvironment.readTextFile("input/mydata.txt");

        // todo 3、切分、转换（word 1）, 基于interface写一个匿名类
        FlatMapOperator<String, Tuple2<String, Integer>> wordAndOne = stringDataSource.flatMap(new FlatMapFunction<String, Tuple2<String, Integer>>() {
            public void flatMap(String value, Collector<Tuple2<String, Integer>> out) throws Exception {
                // todo 3.1 按照空格切分数组
                String[] words = value.split(" ");
                for (String word : words) {
                    Tuple2<String, Integer> wordTuple2 = Tuple2.of(word, 1);
                    // todo 3.2 使用collector 向下下游发送数据
                    out.collect(wordTuple2);
                }
            }
        });

        // todo 4、各分组内聚
        UnsortedGrouping<Tuple2<String, Integer>> wordAndOneGroupBy = wordAndOne.groupBy(0);

        // todo 5、各个分组内聚合
        AggregateOperator<Tuple2<String, Integer>> sum = wordAndOneGroupBy.sum(1); //这里的1是位置

        // todo 6、输出
        sum.print();
    }
}

	<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>com.jackray.soulland</groupId>
    <artifactId>soulland</artifactId>
    <version>1.0-SNAPSHOT</version>

    <properties>
        <flink.version>1.17.0</flink.version>
    </properties>

    <dependencies>
        
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-streaming-java</artifactId>
            <version>${flink.version}</version>
        </dependency>

        <!-- https://mvnrepository.com/artifact/org.apache.flink/flink-clients -->
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-clients</artifactId>
            <version>${flink.version}</version>
        </dependency>


    </dependencies>
</project>