1.Flink快速上手之WordCount

设置依赖
pom.xml

注：flink-streaming-scala_2.12 => org.apache.flink:flink-runtime_2.12:1.12.1 =>
com.typesafe.akka:akka-actor_2.12:2.5.21，akka就是用scala实现的。即使这里我们用java语言，还是用到了scala实现的包

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>org.example</groupId>
    <artifactId>FlinkTutorial</artifactId>
    <version>1.0-SNAPSHOT</version>

    <properties>
        <maven.compiler.source>8</maven.compiler.source>
        <maven.compiler.target>8</maven.compiler.target>
        <flink.version>1.12.1</flink.version>
        <scala.binary.version>2.12</scala.binary.version>
    </properties>
    <dependencies>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-java</artifactId>
            <version>${flink.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-streaming-scala_${scala.binary.version}</artifactId>
            <version>${flink.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-clients_${scala.binary.version}</artifactId>
            <version>${flink.version}</version>
        </dependency>
    </dependencies>
</project>

1 WordCount 单词统计

package com.atguighu.wc;

import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.util.Collector;

/**
 * 描述:
 * 单词统计
 *
 * @outhor lkq
 * @create 2021-05-11 15:26
 */
public class WordCount {
    public static void main(String[] args)throws Exception{
        // 创建执行环境
        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

        //从文件中读取数据
        // 获取路径
        String inputPath = "src/main/resources/hello.txt";
        // 从文件中读取数据
        DataSet<String> inputDataSet  = env.readTextFile(inputPath,"UTF-8");

        //  对数据集进行处理,按空格分词展开，装换成（word，1） 二元组进行统计
        DataSet<Tuple2<String,Integer>> resultSet  =  inputDataSet.flatMap(new MyFlatMapper())   // flatMap 映射
            .groupBy(0)    // 按照第一个位置的word 分组
            .sum(1);   // 将第二个位置上的数据求和
        resultSet.print();

    }
    // 自定义类，实现MyFlatMapper接口
    public static class MyFlatMapper implements FlatMapFunction<String, Tuple2<String,Integer>>{
        @Override
        public void flatMap(String s, Collector<Tuple2<String, Integer>> collector) throws Exception {
            // 分词
            String[] words = s.split(" ");
            // 遍历所有words,包装成二元组进行输出
            for (String word:words){
                // 没有返回类型，因此用out.collect
                collector.collect(new Tuple2<>(word,1));
            }
        }
    }
}

StreamWordCount流式处理单词统计

package com.atguighu.wc;

import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.util.Collector;

/**
 * 描述:
 * 流处理单词统计
 *
 * @outhor lkq
 * @create 2021-05-11 17:13
 */
public class StreamWordCount {
    public static void main(String[] args) throws Exception {
        // 创建流处理执行环境
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        // 加载文件路径
        String inputFilePath = "src/main/resources/hello.txt";
        // 读取数据
        DataStream<String> fileData = env.readTextFile(inputFilePath);
        // 数据进行处理
        DataStream<Tuple2<String,Integer>> resultStream = fileData.flatMap(new WordCount.MyFlatMapper())
                .keyBy(0)
                .sum(1);
        resultStream.print();
        // 执行任务
        env.execute();
    }
}

StreamWordCountTest实时流式数据源测试

windows通过安装netcat软件进行实时通讯下载地址：https://eternallybored.org/misc/netcat/
nc -lp 打开一个socket服务，用于模拟实时的流数据

package com.atguighu.wc;

import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;

/**
 * 描述:
 * 实时流式数据源测试
 *
 * @outhor lkq
 * @create 2021-05-11 21:03
 */
public class StreamWordCountTest {
    public static void main(String[] args) throws Exception{
        // 创建执行环境
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        // 从socket文本流读取数据
        DataStream<String>socketFile = env.socketTextStream("localhost",7777);
        // 基于流数据进行转换计算
        DataStream<Tuple2<String,Integer>> resultData = socketFile.flatMap(new WordCount.MyFlatMapper())
                .keyBy(0)
                .sum(1);
        // 打印
        resultData.print();
        // 执行任务
        env.execute();
    }
}