kafka+storm部署

这里的kafka和storm服务地址都在本地的host文件中生成了映射，在实际使用时还是替换成对应的服务器ip地址

一 使用的jar

在storm中有针对目前常用的数据源的生产者实现

 

二 编写代码

2.1 storm实现

topology编写

public class KafkaAndStormTopologyMain {
    public static void main(String[] args) throws Exception{
        TopologyBuilder topologyBuilder = new TopologyBuilder();
        // "orderMq"消费的topic信息 在zookeeper/myKafka/kafkaSpout下存放该主题的partition 
           信息，以及broker信息
        topologyBuilder.setSpout("kafkaSpout",
                new KafkaSpout(new SpoutConfig(
                        new ZkHosts("zk01:2181,zk02:2181,zk03:2181"),
                        "orderMq",
                        "/myKafka",
                        "kafkaSpout")),1);
        topologyBuilder.setBolt("mybolt1",new ParserOrderMqBolt(),1).shuffleGrouping("kafkaSpout");

        Config config = new Config();
        config.setNumWorkers(1);

        if (args.length>0) {
            StormSubmitter.submitTopology(args[0], config, topologyBuilder.createTopology());
        }else {
            LocalCluster localCluster = new LocalCluster();
            localCluster.submitTopology("storm2kafka", config, topologyBuilder.createTopology());
        }
    }
}

bolt编写

public class ParserOrderMqBolt extends BaseRichBolt {
    private JedisPool pool;
    @Override
    public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
        //change "maxActive" -> "maxTotal" and "maxWait" -> "maxWaitMillis" in all examples
        JedisPoolConfig config = new JedisPoolConfig();
        //控制一个pool最多有多少个状态为idle(空闲的)的jedis实例。
        config.setMaxIdle(5);
        //控制一个pool可分配多少个jedis实例，通过pool.getResource()来获取；
        //如果赋值为-1，则表示不限制；如果pool已经分配了maxActive个jedis实例，则此时pool的状态为exhausted(耗尽)。
        //在borrow一个jedis实例时，是否提前进行validate操作；如果为true，则得到的jedis实例均是可用的；
        config.setMaxTotal(1000 * 100);
        //表示当borrow(引入)一个jedis实例时，最大的等待时间，如果超过等待时间，则直接抛出JedisConnectionException；
        config.setMaxWaitMillis(30);
        config.setTestOnBorrow(true);
        config.setTestOnReturn(true);
        /**
         *如果你遇到 java.net.SocketTimeoutException: Read timed out exception的异常信息
         *请尝试在构造JedisPool的时候设置自己的超时值. JedisPool默认的超时时间是2秒(单位毫秒)
         */
        pool = new JedisPool(config, "127.0.0.1", 6379);
    }

    @Override
    public void execute(Tuple input) {
        Jedis jedis = pool.getResource();
        //获取kafkaSpout发送过来的数据，是一个json
        String string = new String((byte[]) input.getValue(0));
        //解析json
        OrderInfo orderInfo = (OrderInfo) new  Gson().fromJson(string, OrderInfo.class);
        //整个网站，各个业务线，各个品类，各个店铺，各个品牌，每个商品
        //获取整个网站的金额统计指标
//        String totalAmount =  jedis.get("totalAmount");
        jedis.incrBy("totalAmount",orderInfo.getProductPrice());
        //获取商品所属业务线的指标信息
        String bid =  getBubyProductId(orderInfo.getProductId(),"b");
//        String bAmout =  jedis.get(bid+"Amout");
        jedis.incrBy(bid+"Amount",orderInfo.getProductPrice());
        jedis.close();
    }

    private String getBubyProductId(String productId,String type) {
//        key:value
        //index:productID:info---->Map
        //  productId-----<各个业务线，各个品类，各个店铺，各个品牌，每个商品>
        Map<String,String> map =  new HashMap<>();
        map.put("b","3c");
        map.put("c","phone");
        map.put("s","121");
        map.put("p","iphone");
        return map.get(type);
    }
    @Override
    public void declareOutputFields(OutputFieldsDeclarer declarer) {

    }
}

 

 2.2 kafka实现

数据生产者

public class OrderMqSender {
    public static void main(String[] args) {
        String TOPIC = "orderMq";
        Properties props = new Properties();
        props.put("serializer.class", "kafka.serializer.StringEncoder");
        props.put("metadata.broker.list", "kafka01:9092,kafka02:9092,kafka03:9092");
        props.put("request.required.acks", "1");
        props.put("partitioner.class", "kafka.producer.DefaultPartitioner");
        Producer<String, String> producer = new Producer<String, String>(new ProducerConfig(props));
        for (int messageNo = 1; messageNo < 100000; messageNo++) {
            producer.send(new KeyedMessage<String, String>(TOPIC, messageNo + "",new OrderInfo().random() ));
//            try {
//                Thread.sleep(100);
//            } catch (InterruptedException e) {
//                e.printStackTrace();
//            }
        }
    }

}

2.3 javabean对象

/**
 * Describe: 简单的支付信息，实际业务更复杂，比如父子订单，一个订单多个商品，一个支付订单多个订单等情况。
 */
public class OrderInfo implements Serializable {
    private String orderId;//订单编号
    private Date createOrderTime;//订单创建时间
    private String paymentId;//支付编号
    private Date paymentTime;//支付时间
    private String productId;//商品编号
    private String productName;//商品名称
    private long productPrice;//商品价格
    private long promotionPrice;//促销价格
    private String shopId;//商铺编号
    private String shopName;//商铺名称
    private String shopMobile;//商品电话
    private long payPrice;//订单支付价格
    private int num;//订单数量
    public OrderInfo() {
    }
    public OrderInfo(String orderId, Date createOrderTime, String paymentId, Date paymentTime, String productId, String productName, long productPrice, long promotionPrice, String shopId, String shopName, String shopMobile, long payPrice, int num) {
        this.orderId = orderId;
        this.createOrderTime = createOrderTime;
        this.paymentId = paymentId;
        this.paymentTime = paymentTime;
        this.productId = productId;
        this.productName = productName;
        this.productPrice = productPrice;
        this.promotionPrice = promotionPrice;
        this.shopId = shopId;
        this.shopName = shopName;
        this.shopMobile = shopMobile;
        this.payPrice = payPrice;
        this.num = num;
    }
    public String getOrderId() {return orderId;}
    public void setOrderId(String orderId) { this.orderId = orderId;}
    public Date getCreateOrderTime(){return createOrderTime;}
    public void setCreateOrderTime(Date createOrderTime) {this.createOrderTime = 
        createOrderTime;}
    public String getPaymentId() {return paymentId;}
    public void setPaymentId(String paymentId) { this.paymentId = paymentId;}
    public Date getPaymentTime() {return paymentTime;}
    public void setPaymentTime(Date paymentTime) {this.paymentTime = paymentTime;}
    public String getProductId() { return productId;}
    public void setProductId(String productId) {this.productId = productId;}
    public String getProductName() { return productName;}
    public void setProductName(String productName) {this.productName = productName;}
    public long getProductPrice() {return productPrice;}
    public void setProductPrice(long productPrice) {this.productPrice = productPrice;}
    public long getPromotionPrice() {return promotionPrice; }
    public void setPromotionPrice(long promotionPrice) {this.promotionPrice = 
       promotionPrice; }
    public String getShopId() {return shopId;}
    public void setShopId(String shopId) {this.shopId = shopId;}
    public String getShopName() {  return shopName;}
    public void setShopName(String shopName) { this.shopName = shopName;}
    public String getShopMobile() {return shopMobile;}
    public void setShopMobile(String shopMobile) {this.shopMobile = shopMobile;}
    public long getPayPrice() { return payPrice;}
    public void setPayPrice(long payPrice) {this.payPrice = payPrice;}
    public int getNum() {return num; }
    public void setNum(int num) { this.num = num; }
    @Override
    public String toString() {
        return "PaymentInfo{" +
                "orderId='" + orderId + '\'' +
                ", createOrderTime=" + createOrderTime +
                ", paymentId='" + paymentId + '\'' +
                ", paymentTime=" + paymentTime +
                ", productId='" + productId + '\'' +
                ", productName='" + productName + '\'' +
                ", productPrice=" + productPrice +
                ", promotionPrice=" + promotionPrice +
                ", shopId='" + shopId + '\'' +
                ", shopName='" + shopName + '\'' +
                ", shopMobile='" + shopMobile + '\'' +
                ", payPrice=" + payPrice +
                ", num=" + num +
                '}';
    }
    public String random() {
        this.productId = "12121212";
        this.orderId = UUID.randomUUID().toString().replaceAll("-", "");
        this.paymentId = UUID.randomUUID().toString().replaceAll("-", "");
        this.productPrice = new Random().nextInt(1000);
        this.promotionPrice = new Random().nextInt(500);
        this.payPrice = new Random().nextInt(480);

        String date = "2015-11-11 12:22:12";
        SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
        try {
            this.createOrderTime = simpleDateFormat.parse(date);
        } catch (ParseException e) {
            e.printStackTrace();
        }

        return new Gson().toJson(this);
    }
}

三 运行方式

需要开启redis，kafka，storm集群。

kafka直接在开发工具上运行

storm有两种运行方式   

     1 同kafka（本地模式）

     2 打成jar包在storm集群中运行

       storm jar 【jar路径】 【拓扑包名.拓扑类名】 【拓扑名称】

       bin/storm jar examples/storm-starter/storm-starter-topologies-0.9.6.jar storm.starter.WordCountTopology wordcount