Storm ack容错机制案例

Ack机制：

spout 发出一条数据后需要知道数据处理成功和失败的状态，如失败的消息重新发送。
1. 自定义spoout实现BeseRichSpout,重写ack、fai方法。
2. 在自定义的spout 发送数据的时候，需要制定messageid,messageid 是一个Object.
3. 当消息处理成功或者失败之后，Storm框架会将messageid传回来。如果消息要重发，直接通过messageid找到或直接转化成数据内容进行重发。
4. 自定义Bolt实现BaseRichBolt
5. 在bolt的execute中进行两个操作： 发送数据时，需要制定血缘关系，锚点collector.emit(父tupe,new 子Tupe)；当execute处理完成业务逻辑的时候，需要告诉storm框架当前阶段的处理状态collector.ack(tupe)

如果在编写storm程序时，在bolt环节忘记了手动ack或fail，怎么办？

忘了手动ack或fail，storm框架会等待反馈，达超时阈值之后，就直接给fail。

如果在编写storm程序时，bolt环节忘了标示锚点，怎么办？

忘了标示锚点，及时完了标血缘关系。storm会认为你不关系后面阶段的处理。

Storm BaseRichClient Api 可能有点繁琐，可以使用BaseBasicBolt

如果实现了BaseBasicBolt，就不需要锚点，不需要手动ack或fail

---

基于BaseRichSpout、BaseRichBolt

AckSpout:

public class AckSpout extends BaseRichSpout {
    private SpoutOutputCollector collector;

    //初始化方法
    public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
        this.collector = collector;
    }

    //上帝之手，循环调用，每调用过一次就发送一条消息
    public void nextTuple() {
        //生产一条数据
        String uuid = UUID.randomUUID().toString().replace("_", "");
        collector.emit(new Values(uuid),new Values(uuid));
        try {
            Thread.sleep(10*1000);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
    }

    //定义发送的字段
    public void declareOutputFields(OutputFieldsDeclarer declarer) {
        declarer.declare(new Fields("uuid"));
    }

    @Override
    public void ack(Object msgId) {
        System.out.println("消息处理成功" + msgId);
    }

    @Override
    public void fail(Object msgId) {
        System.out.println("消息处理失败" + msgId);
        collector.emit((List)msgId,msgId );
    }
}

Bolt1

public class Bolt1 extends BaseRichBolt {
    private OutputCollector collector;

    //初始化方法 只调用一次
    public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
        this.collector = collector;
    }

    //被循环调用
    public void execute(Tuple input) {
        collector.emit(input,new Values(input.getString(0)));
        System.out.println("bolt1的execute方法被调用一次" + input.getString(0));
        collector.ack(input);
    }

    @Override
    public void declareOutputFields(OutputFieldsDeclarer declarer) {
        declarer.declare(new Fields("uuid"));
    }
}

Bolt2

public class Bolt2 extends BaseRichBolt {
    private OutputCollector collector;

    //初始化方法 只调用一次
    public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
        this.collector = collector;
    }

    //被循环调用
    public void execute(Tuple input) {
        collector.emit(input,new Values(input.getString(0)));
        System.out.println("bolt2的execute方法被调用一次" + input.getString(0));
        collector.ack(input);
    }

    @Override
    public void declareOutputFields(OutputFieldsDeclarer declarer) {
        declarer.declare(new Fields("uuid"));
    }
}

Bolt3

public class Bolt3 extends BaseRichBolt {
    private OutputCollector collector;

    //初始化方法 只调用一次
    public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
        this.collector = collector;
    }

    //被循环调用
    public void execute(Tuple input) {
        collector.emit(input,new Values(input.getString(0)));
        System.out.println("bolt3的execute方法被调用一次" + input.getString(0));
        collector.fail(input);
    }

    @Override
    public void declareOutputFields(OutputFieldsDeclarer declarer) {
        declarer.declare(new Fields("uuid"));
    }
}

Bolt4

public class Bolt4 extends BaseRichBolt {
    private OutputCollector collector;

    //初始化方法 只调用一次
    public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
        this.collector = collector;
    }

    //被循环调用
    public void execute(Tuple input) {
        collector.emit(input,new Values(input.getString(0)));
        System.out.println("bolt4的execute方法被调用一次" + input.getString(0));
        collector.ack(input);
    }

    @Override
    public void declareOutputFields(OutputFieldsDeclarer declarer) {
        declarer.declare(new Fields("uuid"));
    }
}

AckTopologyDriver

public class AckTopologyDriver {
    public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
        //1、准备任务信息
        TopologyBuilder topologyBuilder = new TopologyBuilder();
        topologyBuilder.setSpout("mySpout", new AckSpout(), 1);
        topologyBuilder.setBolt("bolt1", new Bolt1(),1).shuffleGrouping("mySpout");
        topologyBuilder.setBolt("bolt2", new Bolt2(),1).shuffleGrouping("bolt1");
        topologyBuilder.setBolt("bolt3", new Bolt3(),1).shuffleGrouping("bolt2");
        topologyBuilder.setBolt("bolt4", new Bolt4(),1).shuffleGrouping("bolt3");

        Config config = new Config();
        config.setNumWorkers(2);
        StormTopology stormTopology = topologyBuilder.createTopology();
        LocalCluster localCluster = new LocalCluster();
        localCluster.submitTopology("wordcount", config, stormTopology);

    }
}

---

基于BaseRichSpout、BaseBasicBolt

AckSpout

public class AckSpout extends BaseRichSpout {
    private SpoutOutputCollector collector;

    //初始化方法
    public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
        this.collector = collector;
    }

    //上帝之手，循环调用，每调用过一次就发送一条消息
    public void nextTuple() {
        //生产一条数据
        String uuid = UUID.randomUUID().toString().replace("_", "");
        collector.emit(new Values(uuid),new Values(uuid));
        try {
            Thread.sleep(10*1000);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
    }

    //定义发送的字段
    public void declareOutputFields(OutputFieldsDeclarer declarer) {
        declarer.declare(new Fields("uuid"));
    }

    @Override
    public void ack(Object msgId) {
        System.out.println("消息处理成功" + msgId);
    }

    @Override
    public void fail(Object msgId) {
        System.out.println("消息处理失败:重发" + msgId);
        collector.emit((List)msgId,msgId );
    }
}

Bolt1

public class Bolt1 extends BaseBasicBolt {
    @Override
    public void execute(Tuple input, BasicOutputCollector collector) {
        collector.emit(new Values(input.getString(0)));
    }

    @Override
    public void declareOutputFields(OutputFieldsDeclarer declarer) {
        declarer.declare(new Fields("uuid"));
    }
}

Bolt2

public class Bolt2 extends BaseBasicBolt {
    @Override
    public void execute(Tuple input, BasicOutputCollector collector) {
        collector.emit(new Values(input.getString(0)));
    }

    @Override
    public void declareOutputFields(OutputFieldsDeclarer declarer) {
        declarer.declare(new Fields("uuid"));
    }

Bolt3

public class Bolt3 extends BaseBasicBolt {
    @Override
    public void execute(Tuple input, BasicOutputCollector collector) {
//        collector.emit(new Values(input.getString(0)));
        throw new FailedException();
    }

    @Override
    public void declareOutputFields(OutputFieldsDeclarer declarer) {
        declarer.declare(new Fields("uuid"));
    }
}

Bolt4

public class Bolt4 extends BaseBasicBolt {
    @Override
    public void execute(Tuple input, BasicOutputCollector collector) {
        collector.emit(new Values(input.getString(0)));
    }

    @Override
    public void declareOutputFields(OutputFieldsDeclarer declarer) {
        declarer.declare(new Fields("uuid"));
    }
}

AckTopologyDriver

public class AckTopologyDriver {
    public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
        //1、准备任务信息
        TopologyBuilder topologyBuilder = new TopologyBuilder();
        topologyBuilder.setSpout("mySpout", new AckSpout(), 1);
        topologyBuilder.setBolt("bolt1", new Bolt1(), 1).shuffleGrouping("mySpout");
        topologyBuilder.setBolt("bolt2", new Bolt2(), 1).shuffleGrouping("bolt1");
        topologyBuilder.setBolt("bolt3", new Bolt3(), 1).shuffleGrouping("bolt2");
        topologyBuilder.setBolt("bolt4", new Bolt4(), 1).shuffleGrouping("bolt3");

        Config config = new Config();
        config.setNumWorkers(2);
        StormTopology stormTopology = topologyBuilder.createTopology();
        LocalCluster localCluster = new LocalCluster();
        localCluster.submitTopology("wordcount", config, stormTopology);

    }
}