Storm整合jdbc

JDBC的Storm / Trident集成。该软件包包括核心bolts和trident状态，它们允许storm topology.在数据库表中插入strom元组或对数据库执行选择查询，并丰富storm topology.中的元组。
具体核心概念查看：http://storm.apache.org/releases/2.2.0/storm-jdbc.html
4、Storm整合jdbc开发
package com.imooc.bigdata.integration.jdbc;
import com.google.common.collect.Lists;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
import org.apache.storm.jdbc.bolt.JdbcInsertBolt;
import org.apache.storm.jdbc.common.Column;
import org.apache.storm.jdbc.common.ConnectionProvider;
import org.apache.storm.jdbc.common.HikariCPConnectionProvider;
import org.apache.storm.jdbc.mapper.JdbcMapper;
import org.apache.storm.jdbc.mapper.SimpleJdbcMapper;
import org.apache.storm.jdbc.trident.state.JdbcState;
import org.apache.storm.jdbc.trident.state.JdbcStateFactory;
import org.apache.storm.shade.com.google.common.collect.Maps;
import org.apache.storm.spout.SpoutOutputCollector;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.IRichBolt;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.TopologyBuilder;
import org.apache.storm.topology.base.BaseRichBolt;
import org.apache.storm.topology.base.BaseRichSpout;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;
import org.apache.storm.utils.Utils;

import java.sql.Types;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;

/**

• 使用Strom完成词频统计功能
  */

public class LocalWordCountJDBCStormTopology {
/**
* 数据进入
*/
public static class DataSoureceSpout extends BaseRichSpout{
private SpoutOutputCollector collector;

    @Override
    public void open(Map map, TopologyContext topologyContext, SpoutOutputCollector spoutOutputCollector) {
        this.collector=spoutOutputCollector;
    }

    //定义一个字符数组
    public static final String[] words = new String[]{"xiaoqi","xiaoliu","xiaowu","xiaosi","xiaosan","xiaoer","xiaoyi"};

    @Override
    public void nextTuple() {
        //初始化随机函数
        Random random = new Random();
        //随机从words数组里面取出数
        String word = words[random.nextInt(words.length)];
        //发射随机得到的单词
        this.collector.emit(new Values(word));
        //打印发射的内容
        System.out.println("emit: " + word);
        //没发射一个睡一会
        Utils.sleep(1000);

    }

    @Override
    public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
        outputFieldsDeclarer.declare(new Fields("line"));

    }
}


/**
 * 业务：处理数据进行分割
 */
public static class SplitBolt extends BaseRichBolt{
    //定义发射数据
    private OutputCollector collector;

    @Override
    public void prepare(Map map, TopologyContext topologyContext, OutputCollector outputCollector) {
        this.collector=outputCollector;
    }

    /**
     * 业务：处理数据进行分割
     * @param tuple
     */
    @Override
    public void execute(Tuple tuple) {
        String word = tuple.getStringByField("line");
        this.collector.emit(new Values(word));
    }

    @Override
    public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
        outputFieldsDeclarer.declare(new Fields("word"));
    }
}


/**
 * 词频汇总Bolt
 */
public static class CountBolt extends BaseRichBolt{
    private OutputCollector collector;

    @Override
    public void prepare(Map map, TopologyContext topologyContext, OutputCollector outputCollector) {
        this.collector=outputCollector;
    }

    //用来存放获取到的单词
    Map<String,Integer> map = new HashMap<String,Integer>();

    /**
     * 业务逻辑
     * 1、获取每个单词
     * 2、对所有单词进行汇总
     * 3、输出
     * @param tuple
     */
    @Override
    public void execute(Tuple tuple) {
        //1、获取每个单词
        String word = tuple.getStringByField("word");
        //2、判断每个单词是否存在，不存在赋值0，存在加1

// Integer count = map.get(word);
// if (count == null){
// count = 0;
// }
//
// count = 1;
Integer count = 1;

        //单词汇总
        map.put(word,count);

        //输出
        this.collector.emit(new Values(word,map.get(word)));

    }

    @Override
    public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
        outputFieldsDeclarer.declare(new Fields("word","word_count"));
    }
}


/**
 * 集成mysql的存储接口进行处理存储
 */


public static void main(String[] args) {
    //通过TopologyBuilder根据Spout和Bolt构建Topolog
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("DataSoureceSpout",new DataSoureceSpout());
    builder.setBolt("SplitBolt",new SplitBolt()).shuffleGrouping("DataSoureceSpout");
    builder.setBolt("CountBolt",new CountBolt()).shuffleGrouping("SplitBolt");

    Map hikariConfigMap = Maps.newHashMap();
    hikariConfigMap.put("dataSourceClassName","com.mysql.jdbc.jdbc2.optional.MysqlDataSource");
    hikariConfigMap.put("dataSource.url", "jdbc:mysql://192.168.1.14/storm");
    hikariConfigMap.put("dataSource.user","root");
    hikariConfigMap.put("dataSource.password","123456");
    ConnectionProvider connectionProvider = new HikariCPConnectionProvider(hikariConfigMap);

    List<Column> columnSchema = Lists.newArrayList(
            new Column("word", java.sql.Types.VARCHAR),
            new Column("word_count", java.sql.Types.INTEGER));
    JdbcMapper simpleJdbcMapper = new SimpleJdbcMapper(columnSchema);

    JdbcInsertBolt PersistanceBolt = new JdbcInsertBolt(connectionProvider, simpleJdbcMapper)
            .withInsertQuery(" insert into wc(word,word_count) values (?,?) ")
            .withQueryTimeoutSecs(30);

    builder.setBolt("JdbcInsertBolt",PersistanceBolt).shuffleGrouping("CountBolt");


    //创建本地集群
    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("LocalWordCountJDBCStormTopology",new Config(), builder.createTopology());

}

}