【十】storm+mysql集成

使用JdbcInsertBolt、JdbcLookupBolt请直接看官网

官网介绍

这里代码给的例子是wordcount，用的jdbcClient直接执行SQL

maven pom.xml

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  <modelVersion>4.0.0</modelVersion>

  <groupId>com.sid.bigdata</groupId>
  <artifactId>storm</artifactId>
  <version>0.0.1</version>
  <packaging>jar</packaging>

  <name>storm</name>
  <url>http://maven.apache.org</url>

  <properties>
    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    <storm.version>1.1.1</storm.version>
  </properties>

  <dependencies>
    
  <dependency>
  	<groupId>org.apache.storm</groupId>
  	<artifactId>storm-core</artifactId>
  	<version>${storm.version}</version> 
  </dependency>
    <dependency>
  	<groupId>org.apache.storm</groupId>
  	<artifactId>storm-jdbc</artifactId>
  	<version>${storm.version}</version> 
  </dependency>
  
  <dependency>
    <groupId>mysql</groupId>
    <artifactId>mysql-connector-java</artifactId>
    <version>5.1.31</version>
</dependency>
  
  </dependencies>
</project>

spout

package integration.jdbc;

import java.util.Map;
import java.util.Random;

import org.apache.storm.spout.SpoutOutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.base.BaseRichSpout;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Values;
import org.apache.storm.utils.Utils;

/**
 * @author liyijie
 * @date 2018年6月13日下午8:32:24
 * @email 37024760@qq.com
 * @remark
 * @version 
 */
public class WordCountSpout extends BaseRichSpout{

	 private SpoutOutputCollector collector;  
     
     public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {  
         this.collector = collector;  
     }  
     
     public static final String[] words = new String[]{"aaa","bbb","ccc","aa","bb","a"};
     
     /** 
      * 1.把每一行数据发射出去 
      * */  
     public void nextTuple() { 
     	Random random = new Random();
     	String word =words[random.nextInt(words.length)];	                    //获取文件中的每行内容  
         //发射出去  
         this.collector.emit(new Values(word));
         
         System.out.println("emit: "+word);
                   
         Utils.sleep(1000L);   
         }  
      

     public void declareOutputFields(OutputFieldsDeclarer declarer) {  
         declarer.declare(new Fields("word"));  
     }  

}

bolt

package integration.jdbc;

import java.sql.Types;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.storm.jdbc.common.Column;
import org.apache.storm.jdbc.common.ConnectionProvider;
import org.apache.storm.jdbc.common.HikariCPConnectionProvider;
import org.apache.storm.jdbc.common.JdbcClient;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.base.BaseRichBolt;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Tuple;

import com.google.common.collect.Maps;

/**
 * @author liyijie
 * @date 2018年6月13日下午8:58:54
 * @email 37024760@qq.com
 * @remark
 * @version 
 * 
 * 词频汇总Bolt 
 */
public class CountBolt extends BaseRichBolt{

	private OutputCollector collector;
	private JdbcClient jdbcClient;
	private ConnectionProvider connectionProvider;
      
    public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {  
    	this.collector = collector;
    	 Map hikariConfigMap = Maps.newHashMap();
	        hikariConfigMap.put("dataSourceClassName","com.mysql.jdbc.jdbc2.optional.MysqlDataSource");
	        hikariConfigMap.put("dataSource.url", "jdbc:mysql://localhost/sid");
	        hikariConfigMap.put("dataSource.user","root");
	        hikariConfigMap.put("dataSource.password","密码");
	        connectionProvider = new HikariCPConnectionProvider(hikariConfigMap);
	        //对数据库连接池进行初始化
        	connectionProvider.prepare();
	        jdbcClient = new JdbcClient(connectionProvider, 30);
    }  
      
    Map<String,Integer> map = new HashMap<String,Integer>();  
    /** 
     * 业务逻辑 
     * 1.获取每个单词 
     * 2.对所有单词进行汇总 
     * 3.输出 
     * */  
    public void execute(Tuple input) {  
        String word = input.getStringByField("word");  
        Integer count = map.get(word);  
        if(count==null){  
            count=0;  
        }  
            count++;  
          
        map.put(word, count);
       
        //查询该word是否存在
        List<Column> list = new ArrayList();
        //创建一列将值传入   列名  值    值的类型
        list.add(new Column("word", word, Types.VARCHAR)); 
        List<List<Column>> select = jdbcClient.select("select word from wordcount where word = ?",list);
        //计算出查询的条数
        Long n = select.stream().count();  
        if(n>=1){
        	//update
        	jdbcClient.executeSql("update wordcount set word_count = "+map.get(word)+" where word = '"+word+"'");

        }else{
        	//insert
        	jdbcClient.executeSql("insert into wordcount values( '"+word+"',"+map.get(word)+")");

        }
        //collector.emit(new Values(word,map.get(word))); 
    }  

    public void declareOutputFields(OutputFieldsDeclarer declarer) { 
    	//后面jdbc insert bolt直接把这里的输出写Mysql里去了，所以这里的fileds的名字要跟mysql表的字段名字对应
    	declarer.declare(new Fields("word","word_count"));
    }  
    
    @Override
    public void cleanup() {
    	connectionProvider.cleanup();
    } 
}

topology

package integration.jdbc;

import java.sql.Types;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.storm.Config;
import org.apache.storm.LocalCluster;

import org.apache.storm.topology.TopologyBuilder;


/**
 * @author liyijie
 * @date 2018年6月13日上午1:01:08
 * @email 37024760@qq.com
 * @remark
 * @version 
 */
public class LocalWordCountStormJdbcTopology {   
	      
	    public static void main(String[] args) {  
	        //本地模式，没有提交到服务器集群上,不需要搭建storm集群  
	        LocalCluster cluster = new LocalCluster();  
	          
	        //TopologyBuilder根据spout和bolt来构建Topology  
	        //storm中任何一个作业都是通过Topology方式进行提交的  
	        //Topology中需要指定spout和bolt的执行顺序  
	        TopologyBuilder tb = new TopologyBuilder();  
	        tb.setSpout("DataSourceSpout", new WordCountSpout());  
	        //SumBolt以随机分组的方式从DataSourceSpout中接收数据  
	        tb.setBolt("CountBolt", new CountBolt()).shuffleGrouping("DataSourceSpout");  
	  /**
	        Map hikariConfigMap = Maps.newHashMap();
	        hikariConfigMap.put("dataSourceClassName","com.mysql.jdbc.jdbc2.optional.MysqlDataSource");
	        hikariConfigMap.put("dataSource.url", "jdbc:mysql://localhost/sid");
	        hikariConfigMap.put("dataSource.user","root");
	        hikariConfigMap.put("dataSource.password","Liyijie331");
	        ConnectionProvider connectionProvider = new HikariCPConnectionProvider(hikariConfigMap);
	        
	        JdbcClient jdbcClient = new JdbcClient(connectionProvider, 30);
	 */       
	        /**写Mysql
	        //mysql的表名	
	        String tableName = "wordcount";
	        JdbcMapper simpleJdbcMapper = new SimpleJdbcMapper(tableName, connectionProvider);

	        JdbcInsertBolt userPersistanceBolt = new JdbcInsertBolt(connectionProvider, simpleJdbcMapper)
	                                            .withTableName(tableName)
	                                            .withQueryTimeoutSecs(30);
	        
	         tb.setBolt("JdbcInsertBolt", userPersistanceBolt).shuffleGrouping("CountBolt");                            
       
	        */
	     
	        
	        
	        //第一个参数是topology的名称，第三个参数是Topology  
	        cluster.submitTopology("LocalWordCountStormJdbcTopology", new Config(), tb.createTopology());  
	      
	    }  
}

结果