用Flink对Operator State 状态的代码进行hdfs的CheckpointedFunction恢复

最新推荐文章于 2024-05-02 17:53:30 发布

水哥哥的

最新推荐文章于 2024-05-02 17:53:30 发布

阅读量315

点赞数

本文链接：https://blog.csdn.net/fasdfafsd/article/details/100185045

版权

依赖

  <dependencies>

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-core</artifactId>
            <version>1.8.1</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-clients_2.11</artifactId>
            <version>1.8.1</version>
        </dependency>

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-scala_2.11</artifactId>
            <version>1.8.1</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-streaming-scala_2.11</artifactId>
            <version>1.8.1</version>
        </dependency>
        <dependency>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-log4j12</artifactId>
            <version>1.7.26</version>
        </dependency>
        <!--添加HDFS支持-->

        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-hdfs</artifactId>
            <version>2.9.2</version>
        </dependency>

        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-common</artifactId>
            <version>2.9.2</version>
        </dependency>

        <!--添加Kafka支持-->
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-connector-kafka_2.11</artifactId>
            <version>1.8.1</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-json</artifactId>
            <version>1.8.1</version>
        </dependency>

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-connector-filesystem_2.11</artifactId>
            <version>1.8.1</version>
        </dependency>
        <!--写到Redis中-->
        <dependency>
            <groupId>org.apache.bahir</groupId>
            <artifactId>flink-connector-redis_2.11</artifactId>
            <version>1.0</version>
        </dependency>

    </dependencies>
    <build>
        <plugins>
            <!--在执行package时候，将scala源码编译进jar-->
            <plugin>
                <groupId>net.alchim31.maven</groupId>
                <artifactId>scala-maven-plugin</artifactId>
                <version>4.0.1</version>
                <executions>
                    <execution>
                        <id>scala-compile-first</id>
                        <phase>process-resources</phase>
                        <goals>
                            <goal>add-source</goal>
                            <goal>compile</goal>
                        </goals>
                    </execution>
                </executions>
            </plugin>
        </plugins>
    </build>

代码

package com.baizhi.demo05

import org.apache.flink.streaming.api.scala.{StreamExecutionEnvironment, _}

object Flink {
  def main(args: Array[String]): Unit = {
      val spark = StreamExecutionEnvironment.getExecutionEnvironment
        spark.socketTextStream("Flink",9999)
          .flatMap(_.split("\\s+"))
          .map((_,1))
          .keyBy(0)
          .sum(1)
          .addSink(new BufferSink(10))
          spark.execute()
  }
}

package com.baizhi.demo05

import org.apache.flink.api.common.state.{ListState, ListStateDescriptor}
import org.apache.flink.runtime.state.{FunctionInitializationContext, FunctionSnapshotContext}
import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction
import org.apache.flink.streaming.api.functions.sink.SinkFunction
import org.apache.flink.streaming.api.scala._
import scala.collection.JavaConverters._

import scala.collection.mutable.ListBuffer
class BufferSink(threshold: Int = 0) extends SinkFunction[(String,Int)] with CheckpointedFunction {
  @transient
  private var checkpointedState: ListState[(String, Int)] = _
  private val bufferedElements = ListBuffer[(String, Int)]()

  //将结果写出数据
  override def invoke(value: (String, Int), context: SinkFunction.Context[_]): Unit = {
    bufferedElements += value
    if (bufferedElements.size == threshold) {
      for (element <- bufferedElements) {
        println(element)
      }
      bufferedElements.clear()
    }
  }
  //快照逻辑
  override def snapshotState(context: FunctionSnapshotContext): Unit = {
    checkpointedState.clear()//清除上一次状态
    for (element <- bufferedElements) {
      checkpointedState.add(element)
    }
  }
  //初始化状态
  override def initializeState(context: FunctionInitializationContext): Unit = {
    val descriptor = new ListStateDescriptor[(String, Int)]("buffered-elements", createTypeInformation[(String, Int)])
    checkpointedState = context.getOperatorStateStore.getListState(descriptor)
    if(context.isRestored) {//从状态中恢复
      for(element <- checkpointedState.get().asScala) {
        bufferedElements += element
      }
    }
  }
}