Storm 编程实现CallLog日志统计

pom

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <parent>
        <artifactId>hdfsExample</artifactId>
        <groupId>com.mao</groupId>
        <version>1.0-SNAPSHOT</version>
    </parent>
    <modelVersion>4.0.0</modelVersion>

    <groupId>com.mao</groupId>
    <artifactId>stormDemo</artifactId>

    <dependencies>
        <dependency>
            <groupId>org.apache.storm</groupId>
            <artifactId>storm-core</artifactId>
            <version>1.0.6</version>
        </dependency>
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>4.11</version>
        </dependency>
    </dependencies>

</project>

创建Spout

package com.mao.storm;

import org.apache.storm.spout.SpoutOutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.IRichSpout;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Values;

import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Random;

/**
 * Spout类,负责产生数据流
 */
public class CallLogSpout implements IRichSpout {

    //spout 输出收集器
    private SpoutOutputCollector collector;
    //是否完成
    private boolean completed = false;
    //上下文
    private TopologyContext context;
    //随机发生器
    private Random randomGenerator = new Random();
    //索引
    private Integer idx = 0;

    public void open(Map map, TopologyContext context, SpoutOutputCollector collector) {
        this.context = context;
        this.collector = collector;
    }

    public void close() {

    }

    public void activate() {

    }

    public void deactivate() {

    }

    /**
     * 下一个元祖
     */
    public void nextTuple() {

        if (this.idx <= 1000){
            List<String> mobileNumbers = new ArrayList<String>();
            mobileNumbers.add("1234123401");
            mobileNumbers.add("1234123402");
            mobileNumbers.add("1234123403");
            mobileNumbers.add("1234123404");

            Integer localIdx = 0;
            while (localIdx++ < 100 && this.idx++ <1000){
                //取出主叫
                String caller = mobileNumbers.get(randomGenerator.nextInt(4));
                //取出被叫
                String callee = mobileNumbers.get(randomGenerator.nextInt(4));
                while (caller == callee){
                    //重新取出被叫
                    callee = mobileNumbers.get(randomGenerator.nextInt(4));
                }
                //模拟通话时长
                Integer duration = randomGenerator.nextInt(60);
                //输出元祖
                this.collector.emit(new Values(caller,callee,duration));
            }
        }
    }

    public void ack(Object o) {

    }

    public void fail(Object o) {

    }

    /**
     * 定义输出的字段名称
     * @param declarer
     */
    public void declareOutputFields(OutputFieldsDeclarer declarer) {
        declarer.declare(new Fields("from","to","duration"));
    }

    public Map<String, Object> getComponentConfiguration() {
        return null;
    }
}

创建CreatorBolt

package com.mao.storm;

import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.IRichBolt;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;

import java.util.Map;

/**
 * 创建calllog日志的blot
 */
public class CallLogCreatorBlot implements IRichBolt {

    private OutputCollector collector;
    public void prepare(Map map, TopologyContext topologyContext, OutputCollector collector) {
        this.collector = collector;
    }

    public void execute(Tuple tuple) {
        //处理新的通话记录
        String from = tuple.getString(0);
        String to = tuple.getString(1);
        Integer duration = tuple.getInteger(2);
        //产生新的tuple
        collector.emit(new Values(from+" - "+to,duration));
    }

    public void cleanup() {

    }

    /**
     * 设置输出字段的名称
     * @param declarer
     */
    public void declareOutputFields(OutputFieldsDeclarer declarer) {
        declarer.declare(new Fields("call","duration"));
   }

    public Map<String, Object> getComponentConfiguration() {
        return null;
    }
}

创建CounterBolt

package com.mao.storm;

import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.IRichBolt;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Tuple;

import java.util.HashMap;
import java.util.Map;

/**
 * 通话记录计数器Bolt
 */
public class CallLogCounterBolt implements IRichBolt {

    Map<String,Integer> counterMap;

    private OutputCollector collector;

    public void prepare(Map map, TopologyContext topologyContext, OutputCollector collector) {
        this.counterMap = new HashMap<String, Integer>();
        this.collector = collector;
    }

    public void execute(Tuple tuple) {
        String call = tuple.getString(0);
        Integer duration = tuple.getInteger(1);
        if (!counterMap.containsKey(call)){
            counterMap.put(call,1);
        }else {
            Integer c = counterMap.get(call)+duration;
            counterMap.put(call,c);
        }
        collector.ack(tuple);
    }

    public void cleanup() {
        for (Map.Entry<String,Integer> entry : counterMap.entrySet()){
            System.out.println(entry.getKey()+" : "+entry.getValue());
        }
    }

    public void declareOutputFields(OutputFieldsDeclarer declarer) {
        declarer.declare(new Fields("call"));
    }

    public Map<String, Object> getComponentConfiguration() {
        return null;
    }
}

App

package com.mao.storm;


import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
import org.apache.storm.StormSubmitter;
import org.apache.storm.topology.TopologyBuilder;
import org.apache.storm.tuple.Fields;

/**
 * App
 */
public class App {

    public static void main(String[] args) throws Exception {

        TopologyBuilder builder = new TopologyBuilder();

        //设置spout
        builder.setSpout("spout",new CallLogSpout());
        //设置creator-Bolt
        builder.setBolt("creator-bolt",new CallLogCreatorBlot()).shuffleGrouping("spout");
        //设置countor-Bolt
        builder.setBolt("countor-Bolt",new CallLogCounterBolt()).fieldsGrouping("creator-bolt",new Fields("call"));

        Config config = new Config();
        config.setDebug(true);

    /**
     * 本地模式

            LocalCluster cluster = new LocalCluster();
            cluster.submitTopology("LogAnalyserStorm",config,builder.createTopology());
            Thread.sleep(10000);
             //停止集群
             cluster.shutdown();
    */
        StormSubmitter.submitTopology("mytop",config,builder.createTopology());

    }
}

在集群上部署storm top

将项目生成jar包,并上传服务器，在服务器上运行top            

storm jar xxx.jar com.mao.storm.App