Flink WordCount Java 版本

最新推荐文章于 2024-07-03 00:15:00 发布

响彻天堂丶

最新推荐文章于 2024-07-03 00:15:00 发布

阅读量179

点赞数

分类专栏： Flink 文章标签： java flink 开发语言

本文链接：https://blog.csdn.net/qq_34125999/article/details/128571633

版权

Flink 专栏收录该内容

8 篇文章 1 订阅

订阅专栏

文章展示了如何使用ApacheFlink进行批处理和流处理的WordCount程序。批处理示例通过`ExecutionEnvironment`读取文本文件，而流处理示例则通过`StreamExecutionEnvironment`读取文本文件或从NC（网络控制台）接收数据。程序中包含了数据读取、转换、分组和求和操作。

摘要由CSDN通过智能技术生成

1 pom

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-java</artifactId>
            <version>1.10.1</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-streaming-java_2.12</artifactId>
            <version>1.10.1</version>
        </dependency>
    </dependencies>

2 word.txt

flink flink flink flink flink flink flink flink flink flink flink flink flink
spark spark spark spark spark spark spark spark spark spark spark spark spark spark spark spark
hadoop hadoop hadoop hadoop hadoop hadoop hadoop hadoop hadoop hadoop hadoop
yarn yarn yarn yarn yarn yarn yarn
kafka kafka kafka kafka kafka kafka kafka kafka
flume flume flume flume flume flume

3 WordCount

package com.rosh.flink.wc;

import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.api.java.operators.AggregateOperator;
import org.apache.flink.api.java.operators.DataSource;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.util.Collector;

public class WordCount {

    public static void main(String[] args) throws Exception {
        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        String inputPath = "data/input/word.txt";
        DataSource<String> dataSource = env.readTextFile(inputPath);
        AggregateOperator<Tuple2<String, Integer>> result = dataSource.flatMap(new FlatMapFunction<String, Tuple2<String, Integer>>() {
            @Override
            public void flatMap(String line, Collector<Tuple2<String, Integer>> out) throws Exception {
                String[] words = line.split(" ");
                for (String word : words) {
                    out.collect(new Tuple2<>(word, 1));
                }
            }
        }).groupBy(0).sum(1);

        //打印
        result.print();
    }


}

在这里插入图片描述

4 StreamWordCount

package com.rosh.flink.wc;


import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.util.Collector;

public class StreamWordCount {

    public static void main(String[] args) throws Exception {

        //创建流处理执行环境
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        System.out.println("并行度 :" + env.getParallelism());
        //文件路径
        String inputPath = "data/input/word.txt";
        DataStreamSource<String> dataStreamSource = env.readTextFile(inputPath);
        //转换操作
        SingleOutputStreamOperator<Tuple2<String, Integer>> result = dataStreamSource.flatMap(new FlatMapFunction<String, Tuple2<String, Integer>>() {
                    @Override
                    public void flatMap(String line, Collector<Tuple2<String, Integer>> out) throws Exception {
                        String[] words = line.split(" ");
                        for (String word : words) {
                            out.collect(new Tuple2<>(word, 1));
                        }
                    }
                }).keyBy(0)
                .sum(1);
        //打印
        result.print();

        //执行任务
        env.execute();
    }

}

在这里插入图片描述

5 NC

#win10 执行命令
nc -L -p 9999



package com.rosh.flink.wc;

import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.util.Collector;

public class NCWordCount {


    public static void main(String[] args) throws Exception {

        //创建流处理执行环境
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        System.out.println("并行度 :" + env.getParallelism());
        //NC中读取
        DataStreamSource<String> dataStreamSource = env.socketTextStream("127.0.0.1", 9999);
        //转换操作
        SingleOutputStreamOperator<Tuple2<String, Integer>> result = dataStreamSource.flatMap(new FlatMapFunction<String, Tuple2<String, Integer>>() {
                    @Override
                    public void flatMap(String line, Collector<Tuple2<String, Integer>> out) throws Exception {
                        String[] words = line.split(" ");
                        for (String word : words) {
                            out.collect(new Tuple2<>(word, 1));
                        }
                    }
                }).keyBy(0)
                .sum(1);
        //打印
        result.print();

        //执行任务
        env.execute();
    }
}