KafkaSpout源码注解

最新推荐文章于 2023-02-07 17:29:32 发布

HankGuo

最新推荐文章于 2023-02-07 17:29:32 发布

阅读量667

点赞数

本文链接：https://blog.csdn.net/HankGuo/article/details/80067136

版权

本文详细注解了KafkaSpout的源码，包括模块引入、变量定义、构造函数、open函数、关闭连接、tuple处理、ack/fail机制、取消激活函数、output fields声明及emit状态枚举。

摘要由CSDN通过智能技术生成

KafkaSpout源码注释（入门级）

注：本文所有代码块合在一起即为KafkaSpout完整代码（持续更新）

模块引入：

package org.apache.storm.kafka;

import com.google.common.base.Strings;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.storm.kafka.KafkaUtils.KafkaOffsetMetric;
import org.apache.storm.kafka.PartitionManager.KafkaMessageId;
import org.apache.storm.kafka.trident.GlobalPartitionInformation;
import org.apache.storm.metric.api.IMetric;
import org.apache.storm.spout.SpoutOutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.base.BaseRichSpout;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

变量定义

public class KafkaSpout extends BaseRichSpout {
    private static final Logger LOG = LoggerFactory.getLogger(KafkaSpout.class); //指定类的初始化日志对象
    SpoutConfig _spoutConfig; //Spout配置对象
    SpoutOutputCollector _collector; //输出收集器（鬼名自己翻译的）
    PartitionCoordinator _coordinator; //分区协调器
    DynamicPartitionConnections _connections; //动态分区连接
    ZkState _state; //zookeeper连接
    long _lastUpdateMs = 0L; //上次更新时间
    int _currPartitionIndex = 0; //当前分区

构造函数

    public KafkaSpout(SpoutConfig spoutConf) {
        this._spoutConfig = spoutConf; //初始化spout配置
    }

Spout入口open函数

    public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
        this._collector = collector; //初始化收集器
        String topologyInstanceId = context.getStormId(); //获得拓扑名称
        Map<String, Object> stateConf = new HashMap(conf); //用于保存zk的配置信息
        List<String> zkServers = this._spoutConfig.zkServers; //以下获取zk信息，并存入stateConf

        if (zkServers == null) {
            zkServers = (List)conf.get("storm.zookeeper.servers");
        }
            
        Integer zkPort = this._spoutConfig.zkPort;
        if (zkPort == null) {
            zkPort = ((Number)conf.get("storm.zookeeper.port")).intValue();
        }

        stateConf.put("transactional.zookeeper.servers", zkServers);
        stateConf.put("transactional.zookeeper.port", zkPort);
        stateConf.put("transactional.zookeeper.root", this._spoutConfig.zkRoot);
        this._state = new ZkState(stateConf); //初始化zk连接器
        
        //初始化动态分区连接
        this._connections = new DynamicPartitionConnections(this._spoutConfig, KafkaUtils.makeBrokerReader(conf, this._spoutConfig));
        
        //获得spout的数量，在TopologyBuilder.