【九】storm+redis集成

redis安装

官网介绍

测试一下storm写redis。

具体功能描述：随机生成单词，单词计数(word count)，结果写道redis中。

maven pom.xml

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  <modelVersion>4.0.0</modelVersion>

  <groupId>com.sid.bigdata</groupId>
  <artifactId>storm</artifactId>
  <version>0.0.1</version>
  <packaging>jar</packaging>

  <name>storm</name>
  <url>http://maven.apache.org</url>

  <properties>
    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    <storm.version>1.1.1</storm.version>
  </properties>

  <dependencies>
    
  <dependency>
  	<groupId>org.apache.storm</groupId>
  	<artifactId>storm-core</artifactId>
  	<version>${storm.version}</version> 
  </dependency>
  
  <dependency>
  	<groupId>org.apache.storm</groupId>
  	<artifactId>storm-redis</artifactId>
  	<version>${storm.version}</version> 
  </dependency>
  
  </dependencies>
</project>

代码

package integration.redis;

import java.util.HashMap;
import java.util.Map;
import java.util.Random;

import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
import org.apache.storm.redis.bolt.RedisStoreBolt;
import org.apache.storm.redis.common.config.JedisPoolConfig;
import org.apache.storm.redis.common.mapper.RedisDataTypeDescription;
import org.apache.storm.redis.common.mapper.RedisStoreMapper;
import org.apache.storm.spout.SpoutOutputCollector;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.TopologyBuilder;
import org.apache.storm.topology.base.BaseRichBolt;
import org.apache.storm.topology.base.BaseRichSpout;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.ITuple;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;
import org.apache.storm.utils.Utils;

/**
 * @author liyijie
 * @date 2018年6月13日上午1:01:08
 * @email 37024760@qq.com
 * @remark
 * @version 
 */
public class LocalWordCountStormRedisTopology {
	 public static class DataSourceSpout extends BaseRichSpout{  
		  
	        private SpoutOutputCollector collector;  
	          
	        public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {  
	            this.collector = collector;  
	        }  
	        
	        public static final String[] words = new String[]{"aaa","bbb","ccc","aa","bb","a"};
	        
	        /** 
	         * 1.把每一行数据发射出去 
	         * */  
	        public void nextTuple() { 
	        	Random random = new Random();
	        	String word =words[random.nextInt(words.length)];	                    //获取文件中的每行内容  
	            //发射出去  
	            this.collector.emit(new Values(word));
	            
	            System.out.println("emit: "+word);
	                      
	            Utils.sleep(1000L);   
	            }  
	         
	  
	        public void declareOutputFields(OutputFieldsDeclarer declarer) {  
	            declarer.declare(new Fields("word"));  
	        }  
	          
	    }    
	      
	    /** 
	     * 词频汇总Bolt 
	     * */  
	    public static class CountBolt extends BaseRichBolt{  
	    	
	    	private OutputCollector collector;
	          
	        public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {  
	        	this.collector = collector;
	        }  
	          
	        Map<String,Integer> map = new HashMap<String,Integer>();  
	        /** 
	         * 业务逻辑 
	         * 1.获取每个单词 
	         * 2.对所有单词进行汇总 
	         * 3.输出 
	         * */  
	        public void execute(Tuple input) {  
	            String word = input.getStringByField("word");  
	            Integer count = map.get(word);  
	            if(count==null){  
	                count=0;  
	            }  
	                count++;  
	              
	            map.put(word, count);  
	              
	            collector.emit(new Values(word,map.get(word))); 
	        }  
	  
	        public void declareOutputFields(OutputFieldsDeclarer declarer) { 
	        	declarer.declare(new Fields("word","count"));
	        }  
	          
	    }  
	    
	    public static class WordCountStoreMapper implements RedisStoreMapper {
	        private RedisDataTypeDescription description;
	        private final String hashKey = "wordCount";

	        public WordCountStoreMapper() {
	            description = new RedisDataTypeDescription(
	                RedisDataTypeDescription.RedisDataType.HASH, hashKey);
	        }

	        public RedisDataTypeDescription getDataTypeDescription() {
	            return description;
	        }

	        public String getKeyFromTuple(ITuple tuple) {
	            return tuple.getStringByField("word");
	        }

	        public String getValueFromTuple(ITuple tuple) {
	            return tuple.getIntegerByField("count").toString();
	        }
	    }
	      
	    public static void main(String[] args) {  
	        //本地模式，没有提交到服务器集群上,不需要搭建storm集群  
	        LocalCluster cluster = new LocalCluster();  
	          
	        //TopologyBuilder根据spout和bolt来构建Topology  
	        //storm中任何一个作业都是通过Topology方式进行提交的  
	        //Topology中需要指定spout和bolt的执行顺序  
	        TopologyBuilder tb = new TopologyBuilder();  
	        tb.setSpout("DataSourceSpout", new DataSourceSpout());  
	        //SumBolt以随机分组的方式从DataSourceSpout中接收数据  
	        tb.setBolt("CountBolt", new CountBolt()).shuffleGrouping("DataSourceSpout");  
	  
	        //写到redis
	        JedisPoolConfig poolConfig = new JedisPoolConfig.Builder()
	                .setHost("node1").setPort(6379).build();
	        RedisStoreMapper storeMapper = new WordCountStoreMapper();
	        RedisStoreBolt storeBolt = new RedisStoreBolt(poolConfig, storeMapper);
	        tb.setBolt("RedisStoreBolt", storeBolt).shuffleGrouping("CountBolt");
	        
	        //第一个参数是topology的名称，第三个参数是Topology  
	        cluster.submitTopology("LocalWordCountStormRedisTopology", new Config(), tb.createTopology());  
	      
	    }  
}

结果