Flink简单上手wordCount

一、本地环境离线写wordcount
pom.xml配置

  <dependencies>
    <dependency>
      <groupId>org.apache.flink</groupId>
      //源码是由Scala写的，多线程由akka实现
      <artifactId>flink-streaming-java_2.12</artifactId>
      <version>1.10.1</version>
    </dependency>
    <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-java</artifactId>
      <version>1.10.1</version>
    </dependency>
  </dependencies>

代码：

package com.cys.wc;

import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.api.java.operators.DataSource;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.util.Collector;

public class WordCount {
    public static void main(String[] args) throws Exception {
        // 1. 创建执行环境
        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

        //2. 从文件读取数据
        String inputPath = "C:\\Users\\Administrator\\IdeaProjects\\FlinkTutorial\\src\\main\\resources\\hello.txt";
        //离线通过DataSet接收数据，实时通过DataStreaming
        DataSet<String> inputDataSet = env.readTextFile(inputPath);

        //对数据集进行处理，将将数据转换为（work，1）
        //flatMap方法接收一个FlatMapFunction对象，通过这个对象内的flatMap方法处理数据，数据处理结果存储在上下文对象Collector内
        DataSet<Tuple2<String, Integer>> resultSet = inputDataSet.flatMap(new MyFlatMapper())
                .groupBy(0) //按元组第一个位置groupBy，没有reduceby方法
        .sum(1);    //第二个位置上位置求和。

        resultSet.print();

    }

    //自定义类，实现flatMapFunction接口
    public static class MyFlatMapper implements FlatMapFunction<String, Tuple2<String, Integer>> {
        @Override
        //readTextFile是一行一行读的，因此value是一行数据。
        //Collector用于接收数据处理结果
        public void flatMap(String value, Collector<Tuple2<String, Integer>> collector) throws Exception {
            //按空格分割
            String[] words = value.split(" ");
            for(String word: words){
                collector.collect(new Tuple2<String, Integer>(word, 1));
            }
        }
    }
}

二、实时流wordcount

public class StreamingWordCount {
    public static void main(String[] args) throws Exception {
        // 创建流处理执行环境
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

        //设置并行度
        env.setParallelism(2);
        String inputPath = "C:\\Users\\Administrator\\IdeaProjects\\FlinkTutorial\\src\\main\\resources\\hello.txt";

		//区别在于流环境env产生的数据是DataStream
        DataStream<String> inputDataStream = env.readTextFile(inputPath);

        //基于数据流的数据计算
        DataStream<Tuple2<String, Integer>> resultStream = inputDataStream.flatMap(new WordCount.MyFlatMapper())
                .keyBy(0)
                .sum(1);

        //流处理直接调用print将不会有输出结果
        resultStream.print();

        //执行任务
        env.execute();
        /*电脑为8cores，并行度默认为8
        由于流环境处理数据是一个事务处理一次，因此hadoop会有三个结果。
4> (hbase,1)
8> (hadoop,1)
8> (hadoop,2)
8> (hadoop,3)
7> (zookeeper,1)
1> (hive,1)
2> (java,1)
7> (flink,1)
7> (flume,1)
1> (spark,1)
1> (hive,2)*/

    }


}

三、集群提交代码运行

//从socket 端口读取数据
//DataStream<String> inputDataStream = env.socketTextStream("localhost", "7777");

        //用parameter tool工具将参数从args 内提取出来
        ParameterTool parameterTool = ParameterTool.fromArgs(args);
        String hostname = parameterTool.get("hostname");
        int port = parameterTool.getInt("port");
        DataStream<String> inputDataStream = env.socketTextStream(hostname, port);