flink checkpoint恢复测试

测试代码

order 样例类

package com.stanley.checkpoint

/**
  * Created by admin on 2020/7/7.
  */
case class Order(id:Int,category:String,price:Double)

主函数

package com.stanley.checkpoint

import java.util.Properties

import org.apache.flink.api.common.functions.RichMapFunction
import org.apache.flink.api.common.serialization.{SimpleStringSchema}
import org.apache.flink.api.common.state.{ValueState, ValueStateDescriptor}
import org.apache.flink.configuration.Configuration
import org.apache.flink.streaming.api.CheckpointingMode
import org.apache.flink.streaming.api.scala.{DataStream, StreamExecutionEnvironment}
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer011
import org.apache.flink.api.scala._
import org.apache.flink.runtime.state.filesystem.FsStateBackend
import org.apache.flink.streaming.api.environment.CheckpointConfig

/**
  * Created by admin on 2020/7/7.
  */
object CheckPointTest {
  def main(args: Array[String]): Unit = {
    val env = StreamExecutionEnvironment.getExecutionEnvironment
    //激活checkpoint，10秒触发一次
    env.enableCheckpointing(10000L)
    //设置为精确一次，重置kafka偏移量
    env.getCheckpointConfig.setCheckpointingMode(CheckpointingMode.EXACTLY_ONCE)
    //取消任务保留检查点
    env.getCheckpointConfig.enableExternalizedCheckpoints(CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION)
    //设置状态后端路径
    env.setStateBackend(new FsStateBackend("hdfs://node6:8020/user/flink/checkpoint"))
    //kafka配置
    var props = new Properties()
    props.setProperty("bootstrap.servers","node1:9092")
    props.setProperty("group.id","checkpoint2")
    props.setProperty("enable.auto.commit", "true")
    // 偏移量提交的时间间隔，毫秒
    props.setProperty("auto.commit.interval.ms", "5000")
    // kafka 消息的key序列化器
    props.setProperty("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer")
    // kafka 消息的value序列化器
    props.setProperty("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer")
    props.setProperty("auto.offset.reset", "latest")
    //kafka source
    val inputDataStream:DataStream[String] = env.addSource(new FlinkKafkaConsumer011[String]("topic_order",new SimpleStringSchema(),props))
    //输入数据包装成order样例类
    val outputDataStream:DataStream[(Int,String,Double)] = inputDataStream.map(str=>{
      val strs = str.split(",")
      new Order(strs(0).toInt,strs(1),strs(2).toDouble)
    }).keyBy("category").map(new MyMapFunction())
    outputDataStream.print("price")
    env.execute("test")
  }
}

//对不同类别物品价格进行累加
class MyMapFunction extends RichMapFunction[Order,(Int,String,Double)]{
  private var priceState:ValueState[Double]=_
  override def open(parameters: Configuration): Unit = {
    priceState = getRuntimeContext.getState[Double](new ValueStateDescriptor[Double]("price",classOf[Double]))
  }
  override def map(in: Order): (Int,String, Double) = {
    var price= priceState.value()
    price+=in.price
    priceState.update(price)
    (in.id,in.category,price)
  }
}

提交任务

测试

生产数据

控制台输出结果

取消任务

查看checkpoint

从checkpoint恢复任务

继续生产数据

再次取消任务，并从之前的checkpoint恢复