package lantaiyuan.rtscheduling.topology;
import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
import org.apache.storm.StormSubmitter;
import org.apache.storm.generated.AlreadyAliveException;
import org.apache.storm.generated.AuthorizationException;
import org.apache.storm.generated.InvalidTopologyException;
import org.apache.storm.generated.StormTopology;
import org.apache.storm.kafka.KafkaSpout;
import org.apache.storm.kafka.SpoutConfig;
import org.apache.storm.kafka.StringScheme;
import org.apache.storm.kafka.ZkHosts;
import org.apache.storm.spout.SchemeAsMultiScheme;
import org.apache.storm.topology.IRichBolt;
import org.apache.storm.topology.TopologyBuilder;
import org.apache.storm.tuple.Fields;
import org.apache.zookeeper.client.ConnectStringParser;
import lantaiyuan.rtscheduling.bolt.OpDspLineBolt;
import lantaiyuan.rtscheduling.bolt.PassFlowRTUpBolt;
import lantaiyuan.rtscheduling.bolt.PassengerFlowRealTimeUpBolt;
import lantaiyuan.rtscheduling.common.GetRunConfig;
import lantaiyuan.rtscheduling.resbolt.RTLineTopicBolt;
import lantaiyuan.rtscheduling.resbolt.RTLineTopicResultBolt;
/**
* @author Administrator
*实时线路信息
*/
public class RTLineTopicTopo {
public static String zkServer;
public static String spouttopic;
public static String zkroot;
public static String zkid;
public static String redispool;
public static Map<String,String> runConfigMap = new HashMap<String, String>();
public static int NUM_WORKERS;
public static int NUM_ACKERS;
public static int MSG_TIMEOUT;
public static int PASSENGERFLOWREALTIMEUP_BOLT_PARALLELISM_HINT;
public static int OPDSPLINE_BOLT_PARALLELISM_HINT;
public static int FIRST_BOLT_PARALLELISM_HINT;
public static int SECOND_BOLT_PARALLELISM_HINT;
public static int THIRD_BOLT_PARALLELISM_HINT;
//创建topology 读取kafka Topic_FlowStat 这个topic下的数据,进入storm进行计算
public StormTopology buildTopology() {
TopologyBuilder builder = new TopologyBuilder();
builder.setSpout("kafkaSpout_FlowStat", new KafkaSpout(createSpoutConfig("Topic_FlowStat","RTLineTopicTopo_key")), PASSENGERFLOWREALTIMEUP_BOLT_PARALLELISM_HINT);
builder.setBolt("Bolt_FlowStat", new PassFlowRTUpBolt(), PASSENGERFLOWREALTIMEUP_BOLT_PARALLELISM_HINT).shuffleGrouping("kafkaSpout_FlowStat");
builder.setBolt("RTLineTopicResultBolt", new RTLineTopicResultBolt(),THIRD_BOLT_PARALLELISM_HINT).fieldsGrouping("Bolt_FlowStat", new Fields("citycode_lineid"));
return builder.createTopology();
}
/**
* 获取spoutConfig
*
* @param topic
* topic名称
* @param zkRoot
* zk目录
* @param id
* id
* @return
*/
private static SpoutConfig createSpoutConfig(String topic,String id) {
ConnectStringParser connectStringParser = new ConnectStringParser(
zkServer);
List<InetSocketAddress> serverInetAddresses = connectStringParser
.getServerAddresses();
List<String> serverAddresses = new ArrayList<String>(
serverInetAddresses.size());
Integer zkPort = serverInetAddresses.get(0).getPort();
for (InetSocketAddress serverInetAddress : serverInetAddresses) {
serverAddresses.add(serverInetAddress.getHostName());
}
ZkHosts zkHosts = new ZkHosts(zkServer);
SpoutConfig spoutConfig = new SpoutConfig(zkHosts, topic, zkroot, id);
spoutConfig.scheme = new SchemeAsMultiScheme(new StringScheme());
spoutConfig.zkServers = serverAddresses;
spoutConfig.zkPort = zkPort;
spoutConfig.zkRoot = zkroot;
//spoutConfig.ignoreZkOffsets = true;
spoutConfig.startOffsetTime = 0L;
return spoutConfig;
}
public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException, AuthorizationException {
//读取配置文件,加载配置信息
GetRunConfig runConfig = new GetRunConfig();
runConfigMap = runConfig.getStormConf();
NUM_WORKERS = Integer.parseInt(runConfigMap.get("NUM_WORKERS"));
NUM_ACKERS = Integer.parseInt(runConfigMap.get("NUM_ACKERS"));
MSG_TIMEOUT = Integer.parseInt(runConfigMap.get("MSG_TIMEOUT"));
FIRST_BOLT_PARALLELISM_HINT = Integer.parseInt(runConfigMap.get("FIRST_BOLT_PARALLELISM_HINT"));
PASSENGERFLOWREALTIMEUP_BOLT_PARALLELISM_HINT = Integer.parseInt(runConfigMap.get("PASSENGERFLOWREALTIMEUP_BOLT_PARALLELISM_HINT"));
OPDSPLINE_BOLT_PARALLELISM_HINT=Integer.parseInt(runConfigMap.get("OPDSPLINE_BOLT_PARALLELISM_HINT"));
SECOND_BOLT_PARALLELISM_HINT = Integer.parseInt(runConfigMap.get("SECOND_BOLT_PARALLELISM_HINT"));
THIRD_BOLT_PARALLELISM_HINT = Integer.parseInt(runConfigMap.get("THIRD_BOLT_PARALLELISM_HINT"));
zkServer = runConfigMap.get("zkServer");
spouttopic = runConfigMap.get("spouttopic");
zkroot = runConfigMap.get("zkroot");
zkid = runConfigMap.get("zkid");
redispool = runConfigMap.get("redispool");
RTLineTopicTopo kafkaTopology = new RTLineTopicTopo();
StormTopology stormTopology = kafkaTopology.buildTopology();
Config config=new Config();
config.setNumWorkers(NUM_WORKERS);
config.setNumAckers(NUM_ACKERS);
config.setMessageTimeoutSecs(MSG_TIMEOUT);
config.setMaxSpoutPending(5000);
//集群模式提交
if(args.length > 0){
// cluster submit.
try {
StormSubmitter.submitTopology(args[0], config, stormTopology);
} catch (AlreadyAliveException e) {
e.printStackTrace();
} catch (InvalidTopologyException e) {
e.printStackTrace();
}
}else{ //本地模式提交
new LocalCluster().submitTopology(zkid, config, stormTopology);
}
}
}
import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
import org.apache.storm.StormSubmitter;
import org.apache.storm.generated.AlreadyAliveException;
import org.apache.storm.generated.AuthorizationException;
import org.apache.storm.generated.InvalidTopologyException;
import org.apache.storm.generated.StormTopology;
import org.apache.storm.kafka.KafkaSpout;
import org.apache.storm.kafka.SpoutConfig;
import org.apache.storm.kafka.StringScheme;
import org.apache.storm.kafka.ZkHosts;
import org.apache.storm.spout.SchemeAsMultiScheme;
import org.apache.storm.topology.IRichBolt;
import org.apache.storm.topology.TopologyBuilder;
import org.apache.storm.tuple.Fields;
import org.apache.zookeeper.client.ConnectStringParser;
import lantaiyuan.rtscheduling.bolt.OpDspLineBolt;
import lantaiyuan.rtscheduling.bolt.PassFlowRTUpBolt;
import lantaiyuan.rtscheduling.bolt.PassengerFlowRealTimeUpBolt;
import lantaiyuan.rtscheduling.common.GetRunConfig;
import lantaiyuan.rtscheduling.resbolt.RTLineTopicBolt;
import lantaiyuan.rtscheduling.resbolt.RTLineTopicResultBolt;
/**
* @author Administrator
*实时线路信息
*/
public class RTLineTopicTopo {
public static String zkServer;
public static String spouttopic;
public static String zkroot;
public static String zkid;
public static String redispool;
public static Map<String,String> runConfigMap = new HashMap<String, String>();
public static int NUM_WORKERS;
public static int NUM_ACKERS;
public static int MSG_TIMEOUT;
public static int PASSENGERFLOWREALTIMEUP_BOLT_PARALLELISM_HINT;
public static int OPDSPLINE_BOLT_PARALLELISM_HINT;
public static int FIRST_BOLT_PARALLELISM_HINT;
public static int SECOND_BOLT_PARALLELISM_HINT;
public static int THIRD_BOLT_PARALLELISM_HINT;
//创建topology 读取kafka Topic_FlowStat 这个topic下的数据,进入storm进行计算
public StormTopology buildTopology() {
TopologyBuilder builder = new TopologyBuilder();
builder.setSpout("kafkaSpout_FlowStat", new KafkaSpout(createSpoutConfig("Topic_FlowStat","RTLineTopicTopo_key")), PASSENGERFLOWREALTIMEUP_BOLT_PARALLELISM_HINT);
builder.setBolt("Bolt_FlowStat", new PassFlowRTUpBolt(), PASSENGERFLOWREALTIMEUP_BOLT_PARALLELISM_HINT).shuffleGrouping("kafkaSpout_FlowStat");
builder.setBolt("RTLineTopicResultBolt", new RTLineTopicResultBolt(),THIRD_BOLT_PARALLELISM_HINT).fieldsGrouping("Bolt_FlowStat", new Fields("citycode_lineid"));
return builder.createTopology();
}
/**
* 获取spoutConfig
*
* @param topic
* topic名称
* @param zkRoot
* zk目录
* @param id
* id
* @return
*/
private static SpoutConfig createSpoutConfig(String topic,String id) {
ConnectStringParser connectStringParser = new ConnectStringParser(
zkServer);
List<InetSocketAddress> serverInetAddresses = connectStringParser
.getServerAddresses();
List<String> serverAddresses = new ArrayList<String>(
serverInetAddresses.size());
Integer zkPort = serverInetAddresses.get(0).getPort();
for (InetSocketAddress serverInetAddress : serverInetAddresses) {
serverAddresses.add(serverInetAddress.getHostName());
}
ZkHosts zkHosts = new ZkHosts(zkServer);
SpoutConfig spoutConfig = new SpoutConfig(zkHosts, topic, zkroot, id);
spoutConfig.scheme = new SchemeAsMultiScheme(new StringScheme());
spoutConfig.zkServers = serverAddresses;
spoutConfig.zkPort = zkPort;
spoutConfig.zkRoot = zkroot;
//spoutConfig.ignoreZkOffsets = true;
spoutConfig.startOffsetTime = 0L;
return spoutConfig;
}
public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException, AuthorizationException {
//读取配置文件,加载配置信息
GetRunConfig runConfig = new GetRunConfig();
runConfigMap = runConfig.getStormConf();
NUM_WORKERS = Integer.parseInt(runConfigMap.get("NUM_WORKERS"));
NUM_ACKERS = Integer.parseInt(runConfigMap.get("NUM_ACKERS"));
MSG_TIMEOUT = Integer.parseInt(runConfigMap.get("MSG_TIMEOUT"));
FIRST_BOLT_PARALLELISM_HINT = Integer.parseInt(runConfigMap.get("FIRST_BOLT_PARALLELISM_HINT"));
PASSENGERFLOWREALTIMEUP_BOLT_PARALLELISM_HINT = Integer.parseInt(runConfigMap.get("PASSENGERFLOWREALTIMEUP_BOLT_PARALLELISM_HINT"));
OPDSPLINE_BOLT_PARALLELISM_HINT=Integer.parseInt(runConfigMap.get("OPDSPLINE_BOLT_PARALLELISM_HINT"));
SECOND_BOLT_PARALLELISM_HINT = Integer.parseInt(runConfigMap.get("SECOND_BOLT_PARALLELISM_HINT"));
THIRD_BOLT_PARALLELISM_HINT = Integer.parseInt(runConfigMap.get("THIRD_BOLT_PARALLELISM_HINT"));
zkServer = runConfigMap.get("zkServer");
spouttopic = runConfigMap.get("spouttopic");
zkroot = runConfigMap.get("zkroot");
zkid = runConfigMap.get("zkid");
redispool = runConfigMap.get("redispool");
RTLineTopicTopo kafkaTopology = new RTLineTopicTopo();
StormTopology stormTopology = kafkaTopology.buildTopology();
Config config=new Config();
config.setNumWorkers(NUM_WORKERS);
config.setNumAckers(NUM_ACKERS);
config.setMessageTimeoutSecs(MSG_TIMEOUT);
config.setMaxSpoutPending(5000);
//集群模式提交
if(args.length > 0){
// cluster submit.
try {
StormSubmitter.submitTopology(args[0], config, stormTopology);
} catch (AlreadyAliveException e) {
e.printStackTrace();
} catch (InvalidTopologyException e) {
e.printStackTrace();
}
}else{ //本地模式提交
new LocalCluster().submitTopology(zkid, config, stormTopology);
}
}
}