将Storm流计算的结果存储到Hbase上。以 WordCount 为例
准备工作
1. 启动集群
- 启动ZooKeeper集群
$> zkServer.sh start
- 启动hadoop集群(Hbase集群启动前应该保证Hadoop集群启动,并保证namenode为激活状态)
$> start-dfs.sh
- 启动hbase集群
start-hbase.sh
- 启动Storm集群
//启动master
$> storm nimbus
//启动slave
$> storm supervisor
- 创建表名为wordcount,列族为f1的数据表
$hbase shell> create 'ns1:wordcount','f1'
2. maven依赖导入
<dependency>
<groupId>org.apache.storm</groupId>
<artifactId>storm-hbase</artifactId>
<version>1.1.3</version>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<version>1.2.6</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>2.7.6</version>
</dependency>
3. 配置文件导入
将 hbase-site.xml文件 和 hdfs-site.xml文件导入 resources 目录下
编程实现
源头 Spout
public class WordCountSpout implements IRichSpout {
private TopologyContext context;
private SpoutOutputCollector collector;
private List<String> states;
private Random r = new Random();
private int index = 0;
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
this.context = context;
this.collector = collector;
states = new ArrayList<String>();
states.add("hello world tom");
states.add("hello world tomas");
states.add("hello world tomasLee");
states.add("hello world tomson");
}
public void close() {
}
public void activate() {
}
public void deactivate() {
}
public void nextTuple() {
if (index < 3) {
String line = states.get(r.nextInt(4));
collector.emit(new Values(line));
index++;
}
}
public void ack(Object msgId) {
}
public void fail(Object msgId) {
}
public void declareOutputFields(OutputFieldsDeclarer declarer) {
declarer.declare(new Fields("line"));
}
public Map<String, Object> getComponentConfiguration() {
return null;
}
}
分片 Bolt
public class SplitBolt implements IRichBolt {
private TopologyContext context ;
private OutputCollector collector ;
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
this.context = context ;
this.collector = collector ;
}
public void execute(Tuple tuple) {
String line = tuple.getString(0);
String[] arr = line.split(" ");
for(String s : arr){
collector.emit(new Values(s,1));
}
}
public void cleanup() {
}
public void declareOutputFields(OutputFieldsDeclarer declarer) {
declarer.declare(new Fields("word","count"));
}
public Map<String, Object> getComponentConfiguration() {
return null;
}
}
HbaseBolt——写数据到Hbase库中
public class HbaseBolt implements IRichBolt {
private Table table;
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
try {
Configuration conf = HBaseConfiguration.create();
Connection connection = ConnectionFactory.createConnection(conf);
TableName tableName = TableName.valueOf("ns1:wordcount");
table = connection.getTable(tableName);
} catch (IOException e) {
e.printStackTrace();
}
}
public void execute(Tuple tuple) {
//取到单词
String word = tuple.getString(0);
//取到单词数
Integer count = tuple.getInteger(1);
//使用hbase的计数器进行单词统计
byte[] rowKey = Bytes.toBytes(word);
byte[] f = Bytes.toBytes("f1");
byte[] c = Bytes.toBytes("count");
try {
table.incrementColumnValue(rowKey, f, c, count);
} catch (IOException e) {
e.printStackTrace();
}
}
public void cleanup() {
}
public void declareOutputFields(OutputFieldsDeclarer declarer) {
}
public Map<String, Object> getComponentConfiguration() {
return null;
}
}
WCApp
public class WCApp {
public static void main(String[] args) throws Exception {
TopologyBuilder builder = new TopologyBuilder();
builder.setSpout("wcspout", new WordCountSpout()).setNumTasks(1);
builder.setBolt("split-bolt", new SplitBolt(), 2).shuffleGrouping("wcspout").setNumTasks(2);
builder.setBolt("hbase-bolt", new HbaseBolt(), 2).fieldsGrouping("split-bolt", new Fields("word")).setNumTasks(2);
Config conf = new Config();
LocalCluster cluster = new LocalCluster();
cluster.submitTopology("wc", conf, builder.createTopology());
}
}