Flink编程示例

1. 创建maven项目

• POM文件中添加需要的依赖:

• <?xml version="1.0" encoding="UTF-8"?>
  <project xmlns="http://maven.apache.org/POM/4.0.0"
           xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
           xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
      <modelVersion>4.0.0</modelVersion>
  
      <groupId>com.mischen.it</groupId>
      <artifactId>FlinkDemo</artifactId>
      <version>1.0-SNAPSHOT</version>
  
      <properties>
          <flink.version>1.12.0</flink.version>
          <java.version>1.8</java.version>
          <scala.binary.version>2.11</scala.binary.version>
          <slf4j.version>1.7.30</slf4j.version>
      </properties>
  
      <dependencies>
          <dependency>
              <groupId>org.apache.flink</groupId>
              <artifactId>flink-java</artifactId>
              <version>${flink.version}</version>
              <scope>provided</scope>
          </dependency>
          <dependency>
              <groupId>org.apache.flink</groupId>
              <artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
              <version>${flink.version}</version>
              <scope>provided</scope>
          </dependency>
          <dependency>
              <groupId>org.apache.flink</groupId>
              <artifactId>flink-clients_${scala.binary.version}</artifactId>
              <version>${flink.version}</version>
              <scope>provided</scope>
          </dependency>
  
          <dependency>
              <groupId>org.apache.flink</groupId>
              <artifactId>flink-runtime-web_${scala.binary.version}</artifactId>
              <version>${flink.version}</version>
              <scope>provided</scope>
          </dependency>
  
          <dependency>
              <groupId>org.slf4j</groupId>
              <artifactId>slf4j-api</artifactId>
              <version>${slf4j.version}</version>
              <scope>provided</scope>
          </dependency>
          <dependency>
              <groupId>org.slf4j</groupId>
              <artifactId>slf4j-log4j12</artifactId>
              <version>${slf4j.version}</version>
              <scope>provided</scope>
          </dependency>
          <dependency>
              <groupId>org.apache.logging.log4j</groupId>
              <artifactId>log4j-to-slf4j</artifactId>
              <version>2.14.0</version>
          </dependency>
      </dependencies>
  
      <build>
          <plugins>
              <plugin>
                  <groupId>org.apache.maven.plugins</groupId>
                  <artifactId>maven-assembly-plugin</artifactId>
                  <version>3.3.0</version>
                  <configuration>
                      <descriptorRefs>
                          <descriptorRef>jar-with-dependencies</descriptorRef>
                      </descriptorRefs>
                  </configuration>
                  <executions>
                      <execution>
                          <id>make-assembly</id>
                          <phase>package</phase>
                          <goals>
                              <goal>single</goal>
                          </goals>
                      </execution>
                  </executions>
              </plugin>
          </plugins>
      </build>
  
  
  </project>

  src/main/resources添加文件:log4j.properties

• log4j.rootLogger=error, stdout
  log4j.appender.stdout=org.apache.log4j.ConsoleAppender
  log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
  log4j.appender.stdout.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n

  配置idea, 运行的时候包括provided scope

• 

• 

• 在项目目录下面创建word.txt文件，文件内容如下：

• 

• 创建类：WordCount

• package com.mischen.it;
  
  import org.apache.flink.api.common.functions.FlatMapFunction;
  import org.apache.flink.api.java.ExecutionEnvironment;
  import org.apache.flink.api.java.operators.AggregateOperator;
  import org.apache.flink.api.java.operators.DataSource;
  import org.apache.flink.api.java.operators.FlatMapOperator;
  import org.apache.flink.api.java.operators.UnsortedGrouping;
  import org.apache.flink.api.java.tuple.Tuple2;
  import org.apache.flink.util.Collector;
  
  /**
   * @ClassName WordCount
   * @Description DOTO
   * @Author mischen
   * @Date 2021/6/20 0020 7:54
   * @Version 1.0
   **/
  public class WordCount {
      public static void main(String[] args) throws Exception {
          // 1. 创建执行环境
          ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
         // 2. 从文件读取数据  按行读取(存储的元素就是每行的文本)
          DataSource<String> inputDS = env.readTextFile("input/word.txt");
         // 3. 转换数据格式
          FlatMapOperator<String, Tuple2<String, Long>> wordAndOne =  inputDS.flatMap(new FlatMapFunction<String, Tuple2<String,Long>>() {
              public void flatMap(String value,Collector<Tuple2<String, Long>> out){
                   String[] words = value.split(" ");
                  for (String word : words) {
                      out.collect(Tuple2.of(word, 1L));
                  }
              }
          });
          // 4. 按照 word 进行分组
          UnsortedGrouping<Tuple2<String, Long>> wordAndOneUG = wordAndOne.groupBy(0);
          // 5. 分组内聚合统计
          AggregateOperator<Tuple2<String, Long>> sum = wordAndOneUG.sum(1);
  
         // 6. 打印结果
          sum.print();
  
      }
  }
  
  

  运用main方法，得到运行结果为：

• 

• 从上面的运行结果可以看出来，其实，flink和spark、hadoop其实实现的功能基本都相似，就看你怎么熟练运用各个大数据框架去得到自己想要的数据了