flink学习笔记-flink学习的第一个程序

• 1：创建maven工程，导入jar包

<dependencies>
    <!-- https://mvnrepository.com/artifact/org.apache.flink/flink-streaming-scala -->
    <dependency>
        <groupId>org.apache.flink</groupId>
        <artifactId>flink-streaming-scala_2.11</artifactId>
        <version>1.8.1</version>
    </dependency>

    <dependency>
        <groupId>org.apache.flink</groupId>
        <artifactId>flink-scala_2.11</artifactId>
        <version>1.8.1</version>
    </dependency>

</dependencies>
<build>
    <plugins>
        <!-- 限制jdk版本插件 -->
        <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-compiler-plugin</artifactId>
            <version>3.0</version>
            <configuration>
                <source>1.8</source>
                <target>1.8</target>
                <encoding>UTF-8</encoding>
            </configuration>
        </plugin>
        <!-- 编译scala需要用到的插件 -->
        <plugin>
            <groupId>net.alchim31.maven</groupId>
            <artifactId>scala-maven-plugin</artifactId>
            <version>3.2.2</version>
            <executions>
                <execution>
                    <goals>
                        <goal>compile</goal>
                        <goal>testCompile</goal>
                    </goals>
                </execution>
            </executions>
        </plugin>
        <!-- 项目打包用到的插件 -->
        <plugin>
            <artifactId>maven-assembly-plugin</artifactId>
            <configuration>
                <descriptorRefs>
                    <descriptorRef>jar-with-dependencies</descriptorRef>
                </descriptorRefs>
                <archive>
                    <manifest>
                        <mainClass></mainClass>
                    </manifest>
                </archive>
            </configuration>
            <executions>
                <execution>
                    <id>make-assembly</id>
                    <phase>package</phase>
                    <goals>
                        <goal>single</goal>
                    </goals>
                </execution>
            </executions>
        </plugin>
    </plugins>
</build>

• 2：统计socket当中的单词数量

import org.apache.flink.streaming.api.scala.{DataStream, StreamExecutionEnvironment}
import org.apache.flink.streaming.api.windowing.time.Time


case class CountWord(word:String,count:Long)

object FlinkSocketCount {

  def main(args: Array[String]): Unit = {
    //获取程序入口类
    val environment: StreamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment
    //从socket当中获取数据
    val result: DataStream[String] = environment.socketTextStream("node01",8000)
    //导入隐式转换的包，否则时间不能使用
    import org.apache.flink.api.scala._
    //将数据进行切割，封装到样例类当中，然后进行统计
    val resultValue: DataStream[CountWord] = result
      .flatMap(x => x.split(" "))
      .map(x => CountWord(x,1))
      .keyBy("word")
      .timeWindow(Time.seconds(1),Time.milliseconds(1)) // 按照每秒钟时间窗口，以及每秒钟滑动间隔来进行数据统计
      .sum("count")
    //打印最终输出结果
    resultValue.print().setParallelism(1)
    //启动服务
    environment.execute()
  }
}

• 3: 打包上传到服务器运行
  • 将我们的程序打包，然后上传到服务器进行运行，将我们打包好的程序上传到node01服务器，然后体验在各种模式下进行运行我们的程序(standAlone模式运行程序)
  • 启动flink集群

  cd /opt/install/flink-1.8.1
  bin/start-cluster.sh
  

  • 启动node01的socket服务，并提交flink任务

  nc -lk 8000
  # 打包，并将我们的jar-with-dependencies的jar包上传到node01服务器的/opt路径下
  cd /opt/install/flink-1.8.1/
  bin/flink run --class com.flink.CountWord.FlinkSocketCount /opt/flink-1.0-SNAPSHOT-jar-with-dependencies.jar
  
  

• 4：查询运行结果

cd /opt/install/flink-1.8.1/log
tail -200f flink-hadoop-taskexecutor-1-node01.out