storm和kafka整合案例

package lantaiyuan.rtscheduling.topology;


import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;




import java.util.Map;


import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
import org.apache.storm.StormSubmitter;
import org.apache.storm.generated.AlreadyAliveException;
import org.apache.storm.generated.AuthorizationException;
import org.apache.storm.generated.InvalidTopologyException;
import org.apache.storm.generated.StormTopology;
import org.apache.storm.kafka.KafkaSpout;
import org.apache.storm.kafka.SpoutConfig;
import org.apache.storm.kafka.StringScheme;
import org.apache.storm.kafka.ZkHosts;
import org.apache.storm.spout.SchemeAsMultiScheme;
import org.apache.storm.topology.IRichBolt;
import org.apache.storm.topology.TopologyBuilder;
import org.apache.storm.tuple.Fields;
import org.apache.zookeeper.client.ConnectStringParser;


import lantaiyuan.rtscheduling.bolt.OpDspLineBolt;
import lantaiyuan.rtscheduling.bolt.PassFlowRTUpBolt;
import lantaiyuan.rtscheduling.bolt.PassengerFlowRealTimeUpBolt;
import lantaiyuan.rtscheduling.common.GetRunConfig;
import lantaiyuan.rtscheduling.resbolt.RTLineTopicBolt;
import lantaiyuan.rtscheduling.resbolt.RTLineTopicResultBolt;


/**
 * @author Administrator
 *实时线路信息
 */
public class RTLineTopicTopo {

public static String zkServer;
public static String spouttopic;
public static String zkroot;
public static String zkid;
public static String redispool;
public static Map<String,String> runConfigMap = new HashMap<String, String>();
public static int NUM_WORKERS;  
    public static int NUM_ACKERS;  
    public static int MSG_TIMEOUT;  
    public static int PASSENGERFLOWREALTIMEUP_BOLT_PARALLELISM_HINT;  
    public static int OPDSPLINE_BOLT_PARALLELISM_HINT;
    
    public static int FIRST_BOLT_PARALLELISM_HINT;  
    public static int SECOND_BOLT_PARALLELISM_HINT;  
    public static int THIRD_BOLT_PARALLELISM_HINT;
    //创建topology    读取kafka   Topic_FlowStat  这个topic下的数据，进入storm进行计算
public StormTopology buildTopology() {   
        TopologyBuilder builder = new TopologyBuilder();  
        builder.setSpout("kafkaSpout_FlowStat", new KafkaSpout(createSpoutConfig("Topic_FlowStat","RTLineTopicTopo_key")), PASSENGERFLOWREALTIMEUP_BOLT_PARALLELISM_HINT);  
        builder.setBolt("Bolt_FlowStat", new PassFlowRTUpBolt(), PASSENGERFLOWREALTIMEUP_BOLT_PARALLELISM_HINT).shuffleGrouping("kafkaSpout_FlowStat");  
        builder.setBolt("RTLineTopicResultBolt", new RTLineTopicResultBolt(),THIRD_BOLT_PARALLELISM_HINT).fieldsGrouping("Bolt_FlowStat", new Fields("citycode_lineid"));
        return builder.createTopology();  
    }  
/**
* 获取spoutConfig
*
* @param topic
*            topic名称
* @param zkRoot
*            zk目录
* @param id
*            id
* @return
*/
private static SpoutConfig createSpoutConfig(String topic,String id) {
ConnectStringParser connectStringParser = new ConnectStringParser(
zkServer);
List<InetSocketAddress> serverInetAddresses = connectStringParser
.getServerAddresses();
List<String> serverAddresses = new ArrayList<String>(
serverInetAddresses.size());
Integer zkPort = serverInetAddresses.get(0).getPort();
for (InetSocketAddress serverInetAddress : serverInetAddresses) {
serverAddresses.add(serverInetAddress.getHostName());
}


ZkHosts zkHosts = new ZkHosts(zkServer);
SpoutConfig spoutConfig = new SpoutConfig(zkHosts, topic, zkroot, id);
spoutConfig.scheme = new SchemeAsMultiScheme(new StringScheme());
spoutConfig.zkServers = serverAddresses;
spoutConfig.zkPort = zkPort;
spoutConfig.zkRoot = zkroot;
//spoutConfig.ignoreZkOffsets = true;
spoutConfig.startOffsetTime = 0L;
return spoutConfig;
}

public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException, AuthorizationException {

//读取配置文件，加载配置信息
GetRunConfig runConfig = new GetRunConfig();
runConfigMap = runConfig.getStormConf();
NUM_WORKERS =  Integer.parseInt(runConfigMap.get("NUM_WORKERS"));
NUM_ACKERS =  Integer.parseInt(runConfigMap.get("NUM_ACKERS"));
MSG_TIMEOUT = Integer.parseInt(runConfigMap.get("MSG_TIMEOUT"));
FIRST_BOLT_PARALLELISM_HINT = Integer.parseInt(runConfigMap.get("FIRST_BOLT_PARALLELISM_HINT"));
PASSENGERFLOWREALTIMEUP_BOLT_PARALLELISM_HINT = Integer.parseInt(runConfigMap.get("PASSENGERFLOWREALTIMEUP_BOLT_PARALLELISM_HINT"));
OPDSPLINE_BOLT_PARALLELISM_HINT=Integer.parseInt(runConfigMap.get("OPDSPLINE_BOLT_PARALLELISM_HINT"));
SECOND_BOLT_PARALLELISM_HINT = Integer.parseInt(runConfigMap.get("SECOND_BOLT_PARALLELISM_HINT"));
THIRD_BOLT_PARALLELISM_HINT = Integer.parseInt(runConfigMap.get("THIRD_BOLT_PARALLELISM_HINT"));
zkServer = runConfigMap.get("zkServer");
spouttopic = runConfigMap.get("spouttopic");
zkroot = runConfigMap.get("zkroot");
zkid = runConfigMap.get("zkid");
redispool = runConfigMap.get("redispool");

RTLineTopicTopo kafkaTopology = new RTLineTopicTopo();  
        StormTopology stormTopology = kafkaTopology.buildTopology();  
        
Config config=new Config();
config.setNumWorkers(NUM_WORKERS);  
        config.setNumAckers(NUM_ACKERS);  
        config.setMessageTimeoutSecs(MSG_TIMEOUT);  
        config.setMaxSpoutPending(5000);  

//集群模式提交
        if(args.length > 0){
            // cluster submit.
            try {
                 StormSubmitter.submitTopology(args[0], config, stormTopology);
            } catch (AlreadyAliveException e) {
                e.printStackTrace();
            } catch (InvalidTopologyException e) {
                e.printStackTrace();
            }
        }else{             //本地模式提交
            new LocalCluster().submitTopology(zkid, config, stormTopology);
        }




}

}