storm-kafka联合运用--相关单词统计代码---记录

KafkaTopo
package cn.itcast.storm.topology;

import storm.kafka.BrokerHosts;
import storm.kafka.KafkaSpout;
import storm.kafka.SpoutConfig;
import storm.kafka.ZkHosts;
import backtype.storm.Config;
import backtype.storm.LocalCluster;
import backtype.storm.StormSubmitter;
import backtype.storm.spout.SchemeAsMultiScheme;
import backtype.storm.topology.TopologyBuilder;
import backtype.storm.tuple.Fields;
import cn.itcast.storm.bolt.WordSpliter;
import cn.itcast.storm.bolt.WriterBolt;
import cn.itcast.storm.spout.MessageScheme;

public class KafkaTopo {

	public static void main(String[] args) throws Exception {
		
		String topic = "test1";
		String zkRoot = "/kafka-storm";
		String spoutId = "KafkaSpout";
		BrokerHosts brokerHosts = new ZkHosts("192.168.146.100:2181,192.168.146.101:2181,192.168.146.102:2181"); 
		SpoutConfig spoutConfig = new SpoutConfig(brokerHosts, "test1", zkRoot, spoutId);
		spoutConfig.forceFromStart = true;
		spoutConfig.scheme = new SchemeAsMultiScheme(new MessageScheme());
		TopologyBuilder builder = new TopologyBuilder();
		//设置一个spout用来从kaflka消息队列中读取数据并发送给下一级的bolt组件，此处用的spout组件并非自定义的，而是storm中已经开发好的KafkaSpout
		builder.setSpout("KafkaSpout", new KafkaSpout(spoutConfig));
		builder.setBolt("word-spilter", new WordSpliter()).shuffleGrouping(spoutId);
		builder.setBolt("writer", new WriterBolt(), 4).fieldsGrouping("word-spilter", new Fields("word"));
		Config conf = new Config();
		conf.setNumWorkers(4);
		conf.setNumAckers(0);
		conf.setDebug(false);
		
		//LocalCluster用来将topology提交到本地模拟器运行，方便开发调试
		LocalCluster cluster = new LocalCluster();
		cluster.submitTopology("t", conf, builder.createTopology());
		
		//提交topology到storm集群中运行
//		StormSubmitter.submitTopology("sufei-topo", conf, builder.createTopology());
	}

}

MessageScheme

package cn.itcast.storm.spout;
import java.io.UnsupportedEncodingException;
	
import java.util.List;
	


import backtype.storm.spout.Scheme;

import backtype.storm.tuple.Fields;

import backtype.storm.tuple.Values;



public class MessageScheme implements Scheme {

	

	private static final long serialVersionUID = 8423372426211017613L;



	@Override

	public List<Object> deserialize(byte[] bytes) {

			try {

				String msg = new String(bytes, "UTF-8");

				return new Values(msg); 

			} catch (UnsupportedEncodingException e) {

				e.printStackTrace();

			}

			return null;

	}



	@Override

	public Fields getOutputFields() {

		return new Fields("msg");
		}
			}

WordSpliter

 package cn.itcast.storm.bolt;



import org.apache.commons.lang.StringUtils;



import backtype.storm.topology.BasicOutputCollector;

import backtype.storm.topology.OutputFieldsDeclarer;

import backtype.storm.topology.base.BaseBasicBolt;

import backtype.storm.tuple.Fields;

import backtype.storm.tuple.Tuple;

import backtype.storm.tuple.Values;



public class WordSpliter extends BaseBasicBolt {



	private static final long serialVersionUID = -5653803832498574866L;



	@Override

	public void execute(Tuple input, BasicOutputCollector collector) {

		String line = input.getString(0);

		String[] words = line.split(" ");

		for (String word : words) {

			word = word.trim();

			if (StringUtils.isNotBlank(word)) {

				word = word.toLowerCase();

				collector.emit(new Values(word));

			}

		}

	}



	@Override

	public void declareOutputFields(OutputFieldsDeclarer declarer) {

		declarer.declare(new Fields("word"));
	}

}

WriterBolt

package cn.itcast.storm.bolt;



import java.io.FileWriter;

import java.io.IOException;

import java.util.Map;

import java.util.UUID;



import backtype.storm.task.TopologyContext;

import backtype.storm.topology.BasicOutputCollector;

import backtype.storm.topology.OutputFieldsDeclarer;

import backtype.storm.topology.base.BaseBasicBolt;

import backtype.storm.tuple.Tuple;

/**

 * 将数据写入文件

 * @author duanhaitao@itcast.cn

 *

 */

public class WriterBolt extends BaseBasicBolt {



	private static final long serialVersionUID = -6586283337287975719L;

	

	private FileWriter writer = null;

	

	@Override

	public void prepare(Map stormConf, TopologyContext context) {

		try {

			writer = new FileWriter("/home/hadoop/storm-kafka" + "wordcount"+UUID.randomUUID().toString());

		} catch (IOException e) {

			throw new RuntimeException(e);

		}

	}



	

	@Override

	public void declareOutputFields(OutputFieldsDeclarer declarer) {

	}

	

	

	@Override

	public void execute(Tuple input, BasicOutputCollector collector) {

		String s = input.getString(0);

		try {

			writer.write(s);

			writer.write("\n");

			writer.flush();

		} catch (IOException e) {

			throw new RuntimeException(e);

		}

	}

}

PropertyUtil

package cn.itcast.storm.utils;



import java.io.InputStream;

import java.util.Properties;



import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;



/**

 * 属性配置读取工具

 */

public class PropertyUtil {



	private static final Log log = LogFactory.getLog(PropertyUtil.class);

	private static Properties pros = new Properties();



	// 加载属性文件

	static {

		try {

			InputStream in = PropertyUtil.class.getClassLoader().getResourceAsStream("config.properties");

			pros.load(in);

		} catch (Exception e) {

			log.error("load configuration error", e);

		}

	}



	/**

	 * 读取配置文中的属性值

	 * @param key

	 * @return

	 */

	public static String getProperty(String key) {

		return pros.getProperty(key);

	}



}

来自传智播客视频一位老师的代码