一、源码提取说明
### --- 源码提取说明
~~~ # 源码提取说明:FlinkKafkaConsumerBase.java
~~~ # 第537~693行
@Override
public void open(Configuration configuration) throws Exception {
// determine the offset commit mode
# 指定offset的提交模式: DISABLED、 ON_CHECKPOINTS 、KAFKA_PERIODIC
this.offsetCommitMode = OffsetCommitModes.fromConfiguration(
getIsAutoCommitEnabled(),
enableCommitOnCheckpoints,
((StreamingRuntimeContext) getRuntimeContext()).isCheckpointingEnabled());
// create the partition discoverer
# 创建一个分区发现器
this.partitionDiscoverer = createPartitionDiscoverer(
topicsDescriptor,
getRuntimeContext().getIndexOfThisSubtask(),
getRuntimeContext().getNumberOfParallelSubtasks());
# 实例化出consumer对象
this.partitionDiscoverer.open();
# 已经订阅的分区列表
subscribedPartitionsToStartOffsets = new HashMap<>();
# 获取kafka中的所有分区
final List<KafkaTopicPartition> allPartitions = partitionDiscoverer.discoverPartitions();
if (restoredState != null) {
# restoredState:快照 consumer是从快照中恢复的方式创建
for (KafkaTopicPartition partition : allPartitions) {
if (!restoredState.containsKey(partition)) {
restoredState.put(partition, KafkaTopicPartitionStateSentinel.EARLIEST_OFFSET);
}
}
for (Map.Entry<KafkaTopicPartition, Long> restoredStateEntry : restoredState.entrySet()) {
// seed the partition discoverer with the union state while filtering out
// restored partitions that should not be subscribed by this subtask
# 过滤一下和当前的subTask没有关系的分区数据
if (KafkaTopicPartitionAssigner.assign(
restoredStateEntry.getKey(), getRuntimeContext().getNumberOfParallelSubtasks())
== getRuntimeContext().getIndexOfThisSubtask()){