文章目录
StateBackend创建
StateBackend是state的后端，包括memory、fs、rocksDB
总体来讲，就是根据CheckpointingOptions.STATE_BACKEND创建对应的StateBackend
//StreamTask
//用来创建operatord stateBackend和keyed stateBakend
protected StateBackend stateBackend;
stateBackend = createStateBackend();
//创建checkpoint存储引擎
checkpointStorage = stateBackend.createCheckpointStorage(getEnvironment().getJobID());
private StateBackend createStateBackend() throws Exception {
//从配置中初始化
final StateBackend fromApplication = configuration.getStateBackend(getUserCodeClassLoader());
return StateBackendLoader.fromApplicationOrConfigOrDefault(
fromApplication,
getEnvironment().getTaskManagerInfo().getConfiguration(),
getUserCodeClassLoader(),
LOG);
}
public static StateBackend fromApplicationOrConfigOrDefault(
@Nullable StateBackend fromApplication,
Configuration config,
ClassLoader classLoader,
@Nullable Logger logger) throws IllegalConfigurationException, DynamicCodeLoadingException, IOException {
final StateBackend backend;
// (1) 从配置中初始化StateBackend
if (fromApplication != null) {
if (fromApplication instanceof ConfigurableStateBackend) {
backend = ((ConfigurableStateBackend) fromApplication).configure(config, classLoader);
}
else {
backend = fromApplication;
}
}
else {
// (2) 根据CheckpointingOptions.STATE_BACKEND创建对应的StateBackend
final StateBackend fromConfig = loadStateBackendFromConfig(config, classLoader, logger);
if (fromConfig != null) {
backend = fromConfig;
}
else {
// (3) 默认创建内存的backend
backend = new MemoryStateBackendFactory().createFromConfig(config, classLoader);
if (logger != null) {
logger.info("No state backend has been configured, using default (Memory / JobManager) {}", backend);
}
}
}
//支持旧的方式
if (backend instanceof MemoryStateBackend) {
final MemoryStateBackend memBackend = (MemoryStateBackend) backend;
if (memBackend.getCheckpointPath() == null && HighAvailabilityMode.isHighAvailabilityModeActivated(config)) {
final String haStoragePath = config.getString(HighAvailabilityOptions.HA_STORAGE_PATH);
if (haStoragePath != null) {
try {
Path checkpointDirPath = new Path(haStoragePath, UUID.randomUUID().toString());
if (checkpointDirPath.toUri().getScheme() == null) {
checkpointDirPath = checkpointDirPath.makeQualified(checkpointDirPath.getFileSystem());
}
Configuration tempConfig = new Configuration(config);
tempConfig.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDirPath.toString());
return memBackend.configure(tempConfig, classLoader);
} catch (Exception ignored) {}
}
}
}
return backend;
}
//根据CheckpointingOptions.STATE_BACKEND创建对应的StateBackend
public static StateBackend loadStateBackendFromConfig(
Configuration config,
ClassLoader classLoader,
@Nullable Logger logger) throws IllegalConfigurationException, DynamicCodeLoadingException, IOException {
final String backendName = config.getString(CheckpointingOptions.STATE_BACKEND);
// by default the factory class is the backend name
String factoryClassName = backendName;
switch (backendName.toLowerCase()) {
case MEMORY_STATE_BACKEND_NAME:
MemoryStateBackend memBackend = new MemoryStateBackendFactory().createFromConfig(config, classLoader);
return memBackend;
case FS_STATE_BACKEND_NAME:
FsStateBackend fsBackend = new FsStateBackendFactory().createFromConfig(config, classLoader);
return fsBackend;
case ROCKSDB_STATE_BACKEND_NAME:
factoryClassName = "org.apache.flink.contrib.streaming.state.RocksDBStateBackendFactory";
default:
StateBackendFactory<?> factory;
try {
@SuppressWarnings("rawtypes")
Class<? extends StateBackendFactory> clazz =
Class.forName(factoryClassName, false, classLoader)
.asSubclass(StateBackendFactory.class);
factory = clazz.newInstance();
}
catch (ClassNotFoundException e) {
//exception
}
catch (ClassCastException | InstantiationException | IllegalAccessException e) {
//exception
}
return factory.createFromConfig(config, classLoader);
}
}
//创建CheckpointStreamFactory
private void checkpointState(
CheckpointMetaData checkpointMetaData,
CheckpointOptions checkpointOptions,
CheckpointMetrics checkpointMetrics) throws Exception {
CheckpointStreamFactory storage = checkpointStorage.resolveCheckpointStorageLocation(
checkpointMetaData.getCheckpointId(),
checkpointOptions.getTargetLocation());
CheckpointingOperation checkpointingOperation = new CheckpointingOperation(
this,
checkpointMetaData,
checkpointOptions,
storage,
checkpointMetrics);
checkpointingOperation.executeCheckpointing();
}
public CheckpointStreamFactory resolveCheckpointStorageLocation(
long checkpointId,
CheckpointStorageLocationReference reference) throws IOException {
//checkpointDir file:/H:/flinkTest/checkpoint/ee510cc375b56bea45df694a1bd56f6f/chk-1
if (reference.isDefaultReference()) {
// default reference, construct the default location for that particular checkpoint
final Path checkpointDir = createCheckpointDirectory(checkpointsDirectory, checkpointId);
return new FsCheckpointStorageLocation(
fileSystem,
checkpointDir,
sharedStateDirectory,
taskOwnedStateDirectory,
reference,
fileSizeThreshold);
}
else {
// location encoded in the reference
final Path path = decodePathFromReference(reference);
return new FsCheckpointStorageLocation(
path.getFileSystem(),
path,
path,
path,
reference,
fileSizeThreshold);
}
}
checkpointOptions的初始化
//CheckpointCoordinator
final CheckpointStorageLocation checkpointStorageLocation;
checkpointStorageLocation = props.isSavepoint() ?
checkpointStorage.initializeLocationForSavepoint(checkpointID, externalSavepointLocation) :
checkpointStorage.initializeLocationForCheckpoint(checkpointID);
//FsCheckpointStorage
public CheckpointStorageLocation initializeLocationForCheckpoint(long checkpointId) throws IOException {
checkArgument(checkpointId >= 0);
final Path checkpointDir = createCheckpointDirectory(checkpointsDirectory, checkpointId);
// 创建checkpointDir
fileSystem.mkdirs(checkpointDir);
return new FsCheckpointStorageLocation(
fileSystem,
checkpointDir,
sharedStateDirectory,
taskOwnedStateDirectory,
CheckpointStorageLocationReference.getDefault(),
fileSizeThreshold);
}
state存储
FlinkKafkaConsumerBase是kafka source的基础类，实现了CheckpointListener和CheckpointedFunction这两个接口，通过initializeState来恢复状态，snapshotState来存储状态，notifyCheckpointComplete来完成快照后的回调。这里的状态是operator state，默认的statestore是DefaultOperatorStateBackend。
1.initializeState方法中，会初始化unionOffsetStates
2.首先从registeredOperatorStates中按名字获取partitionableListState，如果为空，则构造一个新的partitionableListState
//FlinkKafkaConsumerBase
//state名称
private static final String OFFSETS_STATE_NAME = "topic-partition-offset-states";
//状态：partition和offset的映射
private transient ListState<Tuple2<KafkaTopicPartition, Long>> unionOffsetStates;
public final void initializeState(FunctionInitializationContext context) throws Exception {
OperatorStateStore stateStore = context.getOperatorStateStore();
//...
this.unionOffsetStates = stateStore.getUnionListState(new ListStateDescriptor<>(
OFFSETS_STATE_NAME,
TypeInformation.of(new TypeHint<Tuple2<KafkaTopicPartition, Long>>() {})));
//...
}
//DefaultOperatorStateBackend
public <S> ListState<S> getUnionListState(ListStateDescriptor<S> stateDescriptor) throws Exception {
return getListState(stateDescriptor, OperatorStateHandle.Mode.UNION);
}
private <S> ListState<S> getListState(
ListStateDescriptor<S> stateDescriptor,
OperatorStateHandle.Mode mode) throws StateMigrationException {
//略
PartitionableListState<S> partitionableListState = (PartitionableListState<S>) registeredOperatorStates.get(name);
if (null == partitionableListState) {
//构造partitionableListState
partitionableListState = new PartitionableListState<>(
new RegisteredOperatorStateBackendMetaInfo<>(
name,
partitionStateSerializer,
mode));
//插入registeredOperatorStates
registeredOperatorStates.put(name, partitionableListState);
} else {
// has restored state; check compatibility of new state access
//略
}
accessedStatesByName.put(name, partitionableListState);
return partitionableListState;
}
3.执行snapshotState，unionOffsetStates中加入当前检查点的状态，于是registeredOperatorStates也保存了此状态
//FlinkKafkaConsumerBase
public final void snapshotState(FunctionSnapshotContext context) throws Exception {
//略
if (fetcher == null) {
//略
} else {
HashMap<KafkaTopicPartition, Long> currentOffsets = fetcher.snapshotCurrentState();
if (offsetCommitMode == OffsetCommitMode.ON_CHECKPOINTS) {
pendingOffsetsToCommit.put(context.getCheckpointId(), currentOffsets);
}
for (Map.Entry<KafkaTopicPartition, Long> kafkaTopicPartitionLongEntry : currentOffsets.entrySet()) {
unionOffsetStates.add(
Tuple2.of(kafkaTopicPartitionLongEntry.getKey(), kafkaTopicPartitionLongEntry.getValue()));
}
}
//略
}
}
}
//AbstractStreamOperator
public final OperatorSnapshotFutures snapshotState(long checkpointId, long timestamp, CheckpointOptions checkpointOptions,
CheckpointStreamFactory factory) throws Exception {
//略
if (null != operatorStateBackend) {
snapshotInProgress.setOperatorStateManagedFuture(
operatorStateBackend.snapshot(checkpointId, timestamp, factory, checkpointOptions));
}
//略
}
//DefaultOperatorStateBackend
public RunnableFuture<SnapshotResult<OperatorStateHandle>> snapshot(
long checkpointId,
long timestamp,
@Nonnull CheckpointStreamFactory streamFactory,
@Nonnull CheckpointOptions checkpointOptions) throws Exception {
long syncStartTime = System.currentTimeMillis();
RunnableFuture<SnapshotResult<OperatorStateHandle>> snapshotRunner =
snapshotStrategy.snapshot(checkpointId, timestamp, streamFactory, checkpointOptions);
snapshotStrategy.logSyncCompleted(streamFactory, syncStartTime);
return snapshotRunner;
}
以下涉及到几个类，来分析下其作用：
CheckpointStateOutputStream：checkpoint输出流，将状态写入文件或内存
StateMetaInfoSnapshot：操作信息，包含名称、状态类型(operator、key-value、broadcast等)、状态类型
DataOutputViewStreamWrapper：包装了CheckpointStateOutputStream，只包含内存，感觉和状态数据视图有关
OperatorStateHandle：operator state操作句柄，可以操作状态
OperatorStateHandle.StateMetaInfo：OperatorStateHandle的元数据，包括offsets和distributionMode(包括SPLIT_DISTRIBUTE、UNION和BROADCAST)
//DefaultOperatorStateBackendSnapshotStrategy
public RunnableFuture<SnapshotResult<OperatorStateHandle>> snapshot(
final long checkpointId,
final long timestamp,
@Nonnull final CheckpointStreamFactory streamFactory,
@Nonnull final CheckpointOptions checkpointOptions) throws IOException {
//如果registeredOperatorStates和registeredBroadcastStates都为空，则返回空
if (registeredOperatorStates.isEmpty() && registeredBroadcastStates.isEmpty()) {
return DoneFuture.of(SnapshotResult.empty());
}
//registeredOperatorStates的拷贝
final Map<String, PartitionableListState<?>> registeredOperatorStatesDeepCopies =
new HashMap<>(registeredOperatorStates.size());
//registeredBroadcastStates的拷贝
final Map<String, BackendWritableBroadcastState<?, ?>> registeredBroadcastStatesDeepCopies =
new HashMap<>(registeredBroadcastStates.size());
ClassLoader snapshotClassLoader = Thread.currentThread().getContextClassLoader();
Thread.currentThread().setContextClassLoader(userClassLoader);
try {
if (!registeredOperatorStates.isEmpty()) {
for (Map.Entry<String, PartitionableListState<?>> entry : registeredOperatorStates.entrySet()) {
PartitionableListState<?> listState = entry.getValue();
if (null != listState) {
//复制一份listState
listState = listState.deepCopy();
}
//插入registeredOperatorStatesDeepCopies
registeredOperatorStatesDeepCopies.put(entry.getKey(), listState);
}
}
//同上复制一份broadcastState插入registeredBroadcastStates
} finally {
Thread.currentThread().setContextClassLoader(snapshotClassLoader);
}
AsyncSnapshotCallable<SnapshotResult<OperatorStateHandle>> snapshotCallable =
new AsyncSnapshotCallable<SnapshotResult<OperatorStateHandle>>() {
@Override
protected SnapshotResult<OperatorStateHandle> callInternal() throws Exception {
//创建一个checkpoint输出流
CheckpointStreamFactory.CheckpointStateOutputStream localOut =
streamFactory.createCheckpointStateOutputStream(CheckpointedStateScope.EXCLUSIVE);
snapshotCloseableRegistry.registerCloseable(localOut);
// 将operator state info插入operatorMetaInfoSnapshots
List<StateMetaInfoSnapshot> operatorMetaInfoSnapshots =
new ArrayList<>(registeredOperatorStatesDeepCopies.size());
for (Map.Entry<String, PartitionableListState<?>> entry :
registeredOperatorStatesDeepCopies.entrySet()) {
operatorMetaInfoSnapshots.add(entry.getValue().getStateMetaInfo().snapshot());
}
// broadcast类似
DataOutputView dov = new DataOutputViewStreamWrapper(localOut);
OperatorBackendSerializationProxy backendSerializationProxy =
new OperatorBackendSerializationProxy(operatorMetaInfoSnapshots, broadcastMetaInfoSnapshots);
//将operatorStateMetaInfoSnapshots和broadcastStateMetaInfoSnapshots写入checkpoint输出流
backendSerializationProxy.write(dov);
//写入operator state和broadcast state的metadata，StateMetaInfo的key是快照名称如：topic-partition-offset-states，value是offset
int initialMapCapacity =
registeredOperatorStatesDeepCopies.size() + registeredBroadcastStatesDeepCopies.size();
final Map<String, OperatorStateHandle.StateMetaInfo> writtenStatesMetaData =
new HashMap<>(initialMapCapacity);
for (Map.Entry<String, PartitionableListState<?>> entry :
registeredOperatorStatesDeepCopies.entrySet()) {
PartitionableListState<?> value = entry.getValue();
long[] partitionOffsets = value.write(localOut);
OperatorStateHandle.Mode mode = value.getStateMetaInfo().getAssignmentMode();
writtenStatesMetaData.put(
entry.getKey(),
new OperatorStateHandle.StateMetaInfo(partitionOffsets, mode));
}
for (Map.Entry<String, BackendWritableBroadcastState<?, ?>> entry :
registeredBroadcastStatesDeepCopies.entrySet()) {
BackendWritableBroadcastState<?, ?> value = entry.getValue();
long[] partitionOffsets = {value.write(localOut)};
OperatorStateHandle.Mode mode = value.getStateMetaInfo().getAssignmentMode();
writtenStatesMetaData.put(
entry.getKey(),
new OperatorStateHandle.StateMetaInfo(partitionOffsets, mode));
}
// 创建一个state handle，可以用来操作状态
OperatorStateHandle retValue = null;
if (snapshotCloseableRegistry.unregisterCloseable(localOut)) {
StreamStateHandle stateHandle = localOut.closeAndGetHandle();
if (stateHandle != null) {
retValue = new OperatorStreamStateHandle(writtenStatesMetaData, stateHandle);
}
return SnapshotResult.of(retValue);
} else {
throw new IOException("Stream was already unregistered.");
}
}
//略
};
final FutureTask<SnapshotResult<OperatorStateHandle>> task =
snapshotCallable.toAsyncSnapshotFutureTask(closeStreamOnCancelRegistry);
if (!asynchronousSnapshots) {
task.run();
}
return task;
}
RocksDBStateBackend
RocksDBStateBackend是唯一支持增量存储状态的后端，state.backend.incremental参数决定检查点是增量还是全量，默认为全量。
RocksDB的架构如下：
Rocksdb中引入了ColumnFamily(列族, CF)的概念，所谓列族也就是一系列kv组成的数据集。所有的读写操作都需要先指定列族。写操作先写WAL，再写memtable，memtable达到一定阈值后切换为Immutable Memtable，只能读不能写。后台Flush线程负责按照时间顺序将Immu Memtable刷盘，生成level0层的有序文件(SST)。后台合并线程负责将上层的SST合并生成下层的SST。Manifest负责记录系统某个时刻SST文件的视图，Current文件记录当前最新的Manifest文件名。 每个ColumnFamily有自己的Memtable， SST文件，所有ColumnFamily共享WAL、Current、Manifest文件。
//RocksDBKeyedStateBackendBuilder.initializeSavepointAndCheckpointStrategies
if (enableIncrementalCheckpointing) {
checkpointSnapshotStrategy = new RocksIncrementalSnapshotStrategy<>(
db,
rocksDBResourceGuard,
keySerializerProvider.currentSchemaSerializer(),
kvStateInformation,
keyGroupRange,
keyGroupPrefixBytes,
localRecoveryConfig,
cancelStreamRegistry,
instanceBasePath,
backendUID,
materializedSstFiles,
lastCompletedCheckpointId,
numberOfTransferingThreads);
} else {
checkpointSnapshotStrategy = savepointSnapshotStrategy;
}
return new SnapshotStrategy<>(checkpointSnapshotStrategy, savepointSnapshotStrategy);
RocksDBStateBackend将in-flight数据存储在RocksDB数据库中，它(默认)存储在TaskManager的data目录下。当checkpoint时，整个RocksDB数据库将被checkpoint到配置的文件系统和目录下。最小的元数据存储在JobManager的内存中(或者，在高可用模式下，在元数据checkpoint中)。
限制：作为RocksDB的JNI桥接API是基于byte[]的，每个key和value的最大的支持大小是 2^31字节。重要：在RocksDB中使用合并操作的状态(例如，ListState)能够默默的积累到值的size大于 2^31字节，并且在下次检索时会失败。这是目前 RocksDB JNI的限制。
object ManagedKeyedStateStreaming {
private val LOG = LoggerFactory.getLogger(ManagedKeyedStateStreaming.getClass)
private val KAFKA_CONSUMER_TOPIC="Test1"
private val KAFKA_BROKERS="node00:9092"
private val KAFKA_ZOOKEEPER_CONNECTION="node00:2181"
private val KAFKA_GROUP_ID="flink-demo-group"
private val KAFKA_PROP: Properties = new Properties() {
setProperty("bootstrap.servers", KAFKA_BROKERS)
setProperty("zookeeper.connect", KAFKA_ZOOKEEPER_CONNECTION)
setProperty("group.id", KAFKA_GROUP_ID)
setProperty("key.serializer", "org.apache.kafka.common.serialization.StringSerializer")
setProperty("value.serializer", "org.apache.kafka.common.serialization.StringSerializer")
}
def main(args: Array[String]): Unit = {
LOG.info("===Stateful Computation Demo===")
val env: StreamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment
env.enableCheckpointing(5000)//5秒一个checkpoint
// env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime)//指定处理的时间特性
env.setRestartStrategy(RestartStrategies.noRestart())//重启策略
env.getCheckpointConfig.setCheckpointingMode(CheckpointingMode.EXACTLY_ONCE)//确保一次语义
// val checkPointPath = new Path("file:\\H:\\flinkTest\\checkpoint")//fs状态后端配置,如为file:///,则在taskmanager的本地
val fsStateBackend: StateBackend= new RocksDBStateBackend("file:\\H:\\flinkTest\\checkpoint")
env.setStateBackend(fsStateBackend)
env.getCheckpointConfig.enableExternalizedCheckpoints(ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION)//退出不删除checkpoint
val dataStream = env.addSource(new FlinkKafkaConsumer010[String](KAFKA_CONSUMER_TOPIC,new SimpleStringSchema(),KAFKA_PROP))
dataStream.filter(_.split("\\|").length==3)
.map(line=>{
val arr = line.split("\\|")
(arr(0),arr(2).toInt)
}).keyBy(_._1)
.flatMap(new SalesAmountCalculation())
.print()
env.execute("Stateful Computation Demo")
}
}
//计算汇总值
class SalesAmountCalculation extends RichFlatMapFunction[(String, Int), (String, Int)] {
private var sum: ValueState[(String, Int)] = _
override def flatMap(input: (String, Int), out: Collector[(String, Int)]): Unit = {
//显式调用已经过期的状态值会被删除，可以配置在读取快照时清除过期状态值，如:
// val ttlConfig = StateTtlConfig
// .newBuilder(Time.seconds(1))
// .cleanupFullSnapshot
// .build
val tmpCurrentSum = sum.value
val currentSum = if (tmpCurrentSum != null) {
tmpCurrentSum
} else {
(input._1, 0)
}
val newSum = (currentSum._1, currentSum._2 + input._2)
sum.update(newSum)
out.collect(newSum)
}
override def open(parameters: Configuration): Unit = {
//设置状态值的过期时间
// val ttlConfig = StateTtlConfig
// .newBuilder(Time.seconds(1))//过期时间1秒
// .setUpdateType(StateTtlConfig.UpdateType.OnCreateAndWrite)//在创建和写入时更新状态值
// .setStateVisibility(StateTtlConfig.StateVisibility.NeverReturnExpired)//过期访问不返回状态值
// .build
val valueStateDescriptor = new ValueStateDescriptor[(String, Int)]("sum", createTypeInformation[(String, Int)])
// valueStateDescriptor.enableTimeToLive(ttlConfig)//启用状态值过期配置
sum = getRuntimeContext.getState(valueStateDescriptor)
}
}
上面的demo是从kafka消费数据，然后对数据每行按照|分割，最后对第一个字段进行分组对第三个字段进行求和。设置了状态后端为RocksDBStateBackend，checkpoint间隔为5s
//AbstractStreamOperator
//此状态数据keyedState，所以走下面的逻辑
if (null != keyedStateBackend) {
snapshotInProgress.setKeyedStateManagedFuture(
keyedStateBackend.snapshot(checkpointId, timestamp, factory, checkpointOptions));
}
//RocksDBKeyedStateBackend
public RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot(
final long checkpointId,
final long timestamp,
@Nonnull final CheckpointStreamFactory streamFactory,
@Nonnull CheckpointOptions checkpointOptions) throws Exception {
long startTime = System.currentTimeMillis();
writeBatchWrapper.flush();
RocksDBSnapshotStrategyBase<K> chosenSnapshotStrategy =
CheckpointType.SAVEPOINT == checkpointOptions.getCheckpointType() ?
savepointSnapshotStrategy : checkpointSnapshotStrategy;
RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshotRunner =
chosenSnapshotStrategy.snapshot(checkpointId, timestamp, streamFactory, checkpointOptions);
return snapshotRunner;
}
//RocksDBSnapshotStrategyBase
public final RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot(
long checkpointId,
long timestamp,
@Nonnull CheckpointStreamFactory streamFactory,
@Nonnull CheckpointOptions checkpointOptions) throws Exception {
if (kvStateInformation.isEmpty()) {
//
return DoneFuture.of(SnapshotResult.empty());
} else {
return doSnapshot(checkpointId, timestamp, streamFactory, checkpointOptions);
}
}
//RocksFullSnapshotStrategy
public RunnableFuture<SnapshotResult<KeyedStateHandle>> doSnapshot(
long checkpointId,
long timestamp,
@Nonnull CheckpointStreamFactory primaryStreamFactory,
@Nonnull CheckpointOptions checkpointOptions) throws Exception {
final SupplierWithException<CheckpointStreamWithResultProvider, Exception> checkpointStreamSupplier =
createCheckpointStreamSupplier(checkpointId, primaryStreamFactory, checkpointOptions);
final List<StateMetaInfoSnapshot> stateMetaInfoSnapshots = new ArrayList<>(kvStateInformation.size());
final List<RocksDbKvStateInfo> metaDataCopy =
new ArrayList<>(kvStateInformation.size());
for (RocksDbKvStateInfo stateInfo : kvStateInformation.values()) {
// snapshot meta info
stateMetaInfoSnapshots.add(stateInfo.metaInfo.snapshot());
metaDataCopy.add(stateInfo);
}
final ResourceGuard.Lease lease = rocksDBResourceGuard.acquireResource();
final Snapshot snapshot = db.getSnapshot();
final SnapshotAsynchronousPartCallable asyncSnapshotCallable =
new SnapshotAsynchronousPartCallable(
checkpointStreamSupplier,
lease,
snapshot,
stateMetaInfoSnapshots,
metaDataCopy,
primaryStreamFactory.toString());
return asyncSnapshotCallable.toAsyncSnapshotFutureTask(cancelStreamRegistry);
}
//RocksFullSnapshotStrategy.SnapshotAsynchronousPartCallable
protected SnapshotResult<KeyedStateHandle> callInternal() throws Exception {
final KeyGroupRangeOffsets keyGroupRangeOffsets = new KeyGroupRangeOffsets(keyGroupRange);
final CheckpointStreamWithResultProvider checkpointStreamWithResultProvider =
checkpointStreamSupplier.get();
snapshotCloseableRegistry.registerCloseable(checkpointStreamWithResultProvider);
writeSnapshotToOutputStream(checkpointStreamWithResultProvider, keyGroupRangeOffsets);
if (snapshotCloseableRegistry.unregisterCloseable(checkpointStreamWithResultProvider)) {
return CheckpointStreamWithResultProvider.toKeyedStateHandleSnapshotResult(
checkpointStreamWithResultProvider.closeAndFinalizeCheckpointStreamResult(),
keyGroupRangeOffsets);
} else {
throw new IOException("Stream is already unregistered/closed.");
}
}
private void writeSnapshotToOutputStream(
@Nonnull CheckpointStreamWithResultProvider checkpointStreamWithResultProvider,
@Nonnull KeyGroupRangeOffsets keyGroupRangeOffsets) throws IOException, InterruptedException {
final List<Tuple2<RocksIteratorWrapper, Integer>> kvStateIterators =
new ArrayList<>(metaData.size());
final DataOutputView outputView =
new DataOutputViewStreamWrapper(checkpointStreamWithResultProvider.getCheckpointOutputStream());
final ReadOptions readOptions = new ReadOptions();
try {
readOptions.setSnapshot(snapshot);
//写入KVState的元数据，包括类型为VALUE、name为sum等信息，写入前先经过序列化。
writeKVStateMetaData(kvStateIterators, readOptions, outputView);
//写入kv数据到数据库
writeKVStateData(kvStateIterators, checkpointStreamWithResultProvider, keyGroupRangeOffsets);
} finally {
//关闭查询
}
}
private void writeKVStateData(
final List<Tuple2<RocksIteratorWrapper, Integer>> kvStateIterators,
final CheckpointStreamWithResultProvider checkpointStreamWithResultProvider,
final KeyGroupRangeOffsets keyGroupRangeOffsets) throws IOException, InterruptedException {
byte[] previousKey = null;
byte[] previousValue = null;
DataOutputView kgOutView = null;
OutputStream kgOutStream = null;
CheckpointStreamFactory.CheckpointStateOutputStream checkpointOutputStream =
checkpointStreamWithResultProvider.getCheckpointOutputStream();
try {
// Here we transfer ownership of RocksIterators to the RocksStatesPerKeyGroupMergeIterator
try (RocksStatesPerKeyGroupMergeIterator mergeIterator = new RocksStatesPerKeyGroupMergeIterator(
kvStateIterators, keyGroupPrefixBytes)) {
//preamble: setup with first key-group as our lookahead
if (mergeIterator.isValid()) {
//begin first key-group by recording the offset
keyGroupRangeOffsets.setKeyGroupOffset(
mergeIterator.keyGroup(),
checkpointOutputStream.getPos());
//write the k/v-state id as metadata
kgOutStream = keyGroupCompressionDecorator.decorateWithCompression(checkpointOutputStream);
kgOutView = new DataOutputViewStreamWrapper(kgOutStream);
//TODO this could be aware of keyGroupPrefixBytes and write only one byte if possible
kgOutView.writeShort(mergeIterator.kvStateId());
previousKey = mergeIterator.key();
previousValue = mergeIterator.value();
mergeIterator.next();
}
//main loop: write k/v pairs ordered by (key-group, kv-state), thereby tracking key-group offsets.
while (mergeIterator.isValid()) {
assert (!hasMetaDataFollowsFlag(previousKey));
//set signal in first key byte that meta data will follow in the stream after this k/v pair
if (mergeIterator.isNewKeyGroup() || mergeIterator.isNewKeyValueState()) {
//be cooperative and check for interruption from time to time in the hot loop
checkInterrupted();
setMetaDataFollowsFlagInKey(previousKey);
}
writeKeyValuePair(previousKey, previousValue, kgOutView);
//write meta data if we have to
if (mergeIterator.isNewKeyGroup()) {
//TODO this could be aware of keyGroupPrefixBytes and write only one byte if possible
kgOutView.writeShort(END_OF_KEY_GROUP_MARK);
// this will just close the outer stream
kgOutStream.close();
//begin new key-group
keyGroupRangeOffsets.setKeyGroupOffset(
mergeIterator.keyGroup(),
checkpointOutputStream.getPos());
//write the kev-state
//TODO this could be aware of keyGroupPrefixBytes and write only one byte if possible
kgOutStream = keyGroupCompressionDecorator.decorateWithCompression(checkpointOutputStream);
kgOutView = new DataOutputViewStreamWrapper(kgOutStream);
kgOutView.writeShort(mergeIterator.kvStateId());
} else if (mergeIterator.isNewKeyValueState()) {
//write the k/v-state
//TODO this could be aware of keyGroupPrefixBytes and write only one byte if possible
kgOutView.writeShort(mergeIterator.kvStateId());
}
//request next k/v pair
previousKey = mergeIterator.key();
previousValue = mergeIterator.value();
mergeIterator.next();
}
}
//epilogue: write last key-group
if (previousKey != null) {
assert (!hasMetaDataFollowsFlag(previousKey));
setMetaDataFollowsFlagInKey(previousKey);
writeKeyValuePair(previousKey, previousValue, kgOutView);
//TODO this could be aware of keyGroupPrefixBytes and write only one byte if possible
kgOutView.writeShort(END_OF_KEY_GROUP_MARK);
// this will just close the outer stream
kgOutStream.close();
kgOutStream = null;
}
} finally {
// this will just close the outer stream
IOUtils.closeQuietly(kgOutStream);
}
SnapshotStategy
快照策略
提供snapshot()方法，给不同的snapshot策略去实现，这里要求snapshot结果返回的类型是RunnableFuture类型
OperatorSnapshotFutures
StreamOperator的snapshotState方法返回值类型为OperatorSnapshotFutures
//StreamOperator
OperatorSnapshotFutures snapshotState(
long checkpointId,
long timestamp,
CheckpointOptions checkpointOptions,
CheckpointStreamFactory storageLocation)
OperatorSnapshotFutures提供了四个属性
| 名称 | 描述 |
|---|---|
| keyedStateManagedFuture | flink管理的keyedState的handle |
| keyedStateRawFuture | 用户管理的keyedState的handle(不推荐) |
| operatorStateManagedFuture | flink管理的operatorState的handle |
| operatorStateRawFuture | 用户管理的keyedState的handle(不推荐) |
//OperatorSnapshotFutures
private RunnableFuture<SnapshotResult<KeyedStateHandle>> keyedStateManagedFuture;
private RunnableFuture<SnapshotResult<KeyedStateHandle>> keyedStateRawFuture;
private RunnableFuture<SnapshotResult<OperatorStateHandle>> operatorStateManagedFuture;
private RunnableFuture<SnapshotResult<OperatorStateHandle>> operatorStateRawFuture;
接下来看下以上四个属性是如何设置的
//AbstreacStreamOperator
snapshotInProgress.setKeyedStateRawFuture(snapshotContext.getKeyedStateStreamFuture());
snapshotInProgress.setOperatorStateRawFuture(snapshotContext.getOperatorStateStreamFuture());
if (null != operatorStateBackend) {
snapshotInProgress.setOperatorStateManagedFuture(
operatorStateBackend.snapshot(checkpointId, timestamp, factory, checkpointOptions));
}
if (null != keyedStateBackend) {
snapshotInProgress.setKeyedStateManagedFuture(
keyedStateBackend.snapshot(checkpointId, timestamp, factory, checkpointOptions));
}
//DefaultOperatorStateBackend
public RunnableFuture<SnapshotResult<OperatorStateHandle>> snapshot(
long checkpointId,
long timestamp,
@Nonnull CheckpointStreamFactory streamFactory,
@Nonnull CheckpointOptions checkpointOptions) throws Exception {
RunnableFuture<SnapshotResult<OperatorStateHandle>> snapshotRunner =
snapshotStrategy.snapshot(checkpointId, timestamp, streamFactory, checkpointOptions);
snapshotStrategy.logSyncCompleted(streamFactory, syncStartTime);
return snapshotRunner;
}
主要来源是从registeredOperatorStates获取。看看registeredOperatorStates是如何产生的
//OperatorStateRestoreOperation
public Void restore() throws Exception {
//遍历stateHandles
for (OperatorStateHandle stateHandle : stateHandles) {
FSDataInputStream in = stateHandle.openInputStream();
closeStreamOnCancelRegistry.registerCloseable(in);
ClassLoader restoreClassLoader = Thread.currentThread().getContextClassLoader();
try {
Thread.currentThread().setContextClassLoader(userClassloader);
OperatorBackendSerializationProxy backendSerializationProxy =
new OperatorBackendSerializationProxy(userClassloader);
backendSerializationProxy.read(new DataInputViewStreamWrapper(in));
//获取restoredOperatorMetaInfoSnapshots，其实就是上一步read(new DataInputViewStreamWrapper(in))的结果
List<StateMetaInfoSnapshot> restoredOperatorMetaInfoSnapshots =
backendSerializationProxy.getOperatorStateMetaInfoSnapshots();
//遍历restoredOperatorMetaInfoSnapshots
for (StateMetaInfoSnapshot restoredSnapshot : restoredOperatorMetaInfoSnapshots) {
//获取restoredMetaInfo
final RegisteredOperatorStateBackendMetaInfo<?> restoredMetaInfo =
new RegisteredOperatorStateBackendMetaInfo<>(restoredSnapshot);
if (restoredMetaInfo.getPartitionStateSerializer() instanceof UnloadableDummyTypeSerializer) {
}
PartitionableListState<?> listState = registeredOperatorStates.get(restoredSnapshot.getName());
if (null == listState) {
//如果listState 为空，用restoredMetaInfo构造一个新的listState
listState = new PartitionableListState<>(restoredMetaInfo);
registeredOperatorStates.put(listState.getStateMetaInfo().getName(), listState);
} else {
}
}
}
CheckpointStreamFactory
创建检查的输出流，用做持久化检查点的数据
最终创建出的流为CheckPointStateOutputStream，继承了FSDataOutputStream，数据最终可以存到磁盘或HDFS。
CompositeStateHandle
恢复
DefaultOperatorStateBackendBuilder.build()—>restoreOperation.restore();
OperatorStateRestoreOperation用于恢复Operator状态
初始化registeredOperatorStates和registeredBroadcastStates
prioritizedOperatorSubtaskStates.getPrioritizedManagedOperatorState()获取stateHandles
stateHandles是OperatorStateHandle的集合
jobManagerState.getManagedKeyedState()
TaskStateSnapshot.subtaskStatesByOperatorID
AsyncCheckpointRunnable.run中对subtaskStatesByOperatorID进行put
//OperatorSnapshotFinalizer
SnapshotResult<KeyedStateHandle> keyedManaged =
FutureUtils.runIfNotDoneAndGet(snapshotFutures.getKeyedStateManagedFuture());
SnapshotResult<KeyedStateHandle> keyedRaw =
FutureUtils.runIfNotDoneAndGet(snapshotFutures.getKeyedStateRawFuture());
SnapshotResult<OperatorStateHandle> operatorManaged =
FutureUtils.runIfNotDoneAndGet(snapshotFutures.getOperatorStateManagedFuture());
SnapshotResult<OperatorStateHandle> operatorRaw =
FutureUtils.runIfNotDoneAndGet(snapshotFutures.getOperatorStateRawFuture());
jobManagerOwnedState = new OperatorSubtaskState(
operatorManaged.getJobManagerOwnedSnapshot(),
operatorRaw.getJobManagerOwnedSnapshot(),
keyedManaged.getJobManagerOwnedSnapshot(),
keyedRaw.getJobManagerOwnedSnapshot()
);
taskLocalState = new OperatorSubtaskState(
operatorManaged.getTaskLocalSnapshot(),
operatorRaw.getTaskLocalSnapshot(),
keyedManaged.getTaskLocalSnapshot(),
keyedRaw.getTaskLocalSnapshot()
);
);
snapshotFutures是operatorSnapshotsInProgress的集合，是对operator执行snapshotState的返回
//AbstractStreamOperator
public final OperatorSnapshotFutures snapshotState(long checkpointId, long timestamp, CheckpointOptions checkpointOptions,
CheckpointStreamFactory factory) throws Exception {
KeyGroupRange keyGroupRange = null != keyedStateBackend ?
keyedStateBackend.getKeyGroupRange() : KeyGroupRange.EMPTY_KEY_GROUP_RANGE;
OperatorSnapshotFutures snapshotInProgress = new OperatorSnapshotFutures();
try (StateSnapshotContextSynchronousImpl snapshotContext = new StateSnapshotContextSynchronousImpl(
checkpointId,
timestamp,
factory,
keyGroupRange,
getContainingTask().getCancelables())) {
snapshotState(snapshotContext);
//设置KeyedStateRawFuture
snapshotInProgress.setKeyedStateRawFuture(snapshotContext.getKeyedStateStreamFuture());
//设置OperatorStateRawFuture
snapshotInProgress.setOperatorStateRawFuture(snapshotContext.getOperatorStateStreamFuture());
if (null != operatorStateBackend) {
//设置OperatorStateManagedFuture
snapshotInProgress.setOperatorStateManagedFuture(
operatorStateBackend.snapshot(checkpointId, timestamp, factory, checkpointOptions));
}
if (null != keyedStateBackend) {
//设置KeyedStateManagedFuture
snapshotInProgress.setKeyedStateManagedFuture(
keyedStateBackend.snapshot(checkpointId, timestamp, factory, checkpointOptions));
}
} catch (Exception snapshotException) {
//异常
}
return snapshotInProgress;
}
407
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
