flume+kafka+storm

最新推荐文章于 2021-01-18 22:50:08 发布

layne_liang

最新推荐文章于 2021-01-18 22:50:08 发布

阅读量292

点赞数

分类专栏： Storm

Storm 专栏收录该内容

5 篇文章 0 订阅

订阅专栏

centos06.6+JDK1.7

flume1.4+kafka2.10+storm0.9.3

zookeeper3.4.6

集群：

192.168.80.133 x01

192.168.80.134 x02

1.两台机器上设置hostname和hosts

。。。

2.两台机器上安装JDK并设置环境变量

3.下载安装zookeeper并设置环境变量

# example sakes.
dataDir=/data/zookeeper/data
# the port at which the clients will connect
clientPort=2181
# the maximum number of client connections.
# increase this if you need to handle more clients
#maxClientCnxns=60
#
# Be sure to read the maintenance section of the
# administrator guide before turning on autopurge.
#
# http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance
#
# The number of snapshots to retain in dataDir
#autopurge.snapRetainCount=3
# Purge task interval in hours
# Set to "0" to disable auto purge feature
#autopurge.purgeInterval=1
server.1=x01:2888:3888
server.2=x02:2888:3888

复制代码

zkServer.sh startzkserver.sh status

4.下载安装flume

http://www.cnblogs.com/admln/p/flume.html

5.下载安装kafka

http://www.cnblogs.com/admln/p/kafka-install.html

6.整合flume和kafka

下载整合插件flumeng-kafka-plugin: https://github.com/beyondj2ee/flumeng-kafka-plugin

提取插件中的flume-conf.properties，修改后放到kafka的conf目录下

############################################
# producer config
###########################################
#agent section
producer.sources = s
producer.channels = c
producer.sinks = r
#source section
producer.sources.s.type = spooldir
producer.sources.s.spoolDir=/home/hadoop/testFlume
producer.sources.s.fileHeader=false
producer.sources.s.channels = c
# Each sink's type must be defined
producer.sinks.r.type = org.apache.flume.plugins.KafkaSink
producer.sinks.r.metadata.broker.list=x01:9092
producer.sinks.r.partition.key=0
producer.sinks.r.partitioner.class=org.apache.flume.plugins.SinglePartition
producer.sinks.r.serializer.class=kafka.serializer.StringEncoder
producer.sinks.r.request.required.acks=0
producer.sinks.r.max.message.size=1000000
producer.sinks.r.producer.type=sync
producer.sinks.r.custom.encoding=UTF-8
producer.sinks.r.custom.topic.name=test
#Specify the channel the sink should use
producer.sinks.r.channel = c
# Each channel's type is defined.
producer.channels.c.type = memory
producer.channels.c.capacity = 1000

复制代码

将Plugin中的jar包拷贝到flume的lib目录中

在/home/hadoop/testFlume中放入文件，在kafka中启用一个console的consumer来测试

bin/flume-ng agent -n producer -c conf -f conf/kafka.conf -Dflume.root.logger=DEBUG,console
bin/kafka-console-consumer.sh --zookeeper x01:2181 --topic test --from-beginning

测试成功

注意：如果让flume传输中文的话，文件编码最好是UTF-8，否则容易乱码导致flume死掉

7.安装storm

http://www.cnblogs.com/admln/p/storm-install.html

8.整合storm和kafka

将kafka的一些jar包复制到storm的lib目录中
cp kafka_2.10-0.8.1.1/libs/kafka_2.10-0.8.1.1.jar apache-storm-0.9.3/lib/
cp kafka_2.10-0.8.1.1/libs/scala-library-2.10.1.jar apache-storm-0.9.3/lib/
cp kafka_2.10-0.8.1.1/libs/metrics-core-2.2.0.jar apache-storm-0.9.3/lib/
cp kafka_2.10-0.8.1.1/libs/snappy-java-1.0.5.jar apache-storm-0.9.3/lib/
cp kafka_2.10-0.8.1.1/libs/zkclient-0.3.jar apache-storm-0.9.3/lib/
cp kafka_2.10-0.8.1.1/libs/log4j-1.2.15.jar apache-storm-0.9.3/lib/
cp kafka_2.10-0.8.1.1/libs/slf4j-api-1.7.2.jar apache-storm-0.9.3/lib/
cp kafka_2.10-0.8.1.1/libs/jopt-simple-3.2.jar apache-storm-0.9.3/lib/
把zookeeper的zookeeper-3.4.6.jar复制到storm的lib目录中
cp zookeeper-3.4.6/zookeeper-3.4.6.jar apache-storm-0.9.3/lib/
编写storm程序来测试
pom.xml
<dependencies>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>3.8.1</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.storm</groupId>
<artifactId>storm-core</artifactId>
<version>0.9.3</version>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka_2.10</artifactId>
<version>0.8.1.1</version>
<exclusions>
<exclusion>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</exclusion>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
spout
package org.admln.flume_kafka_storm;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import kafka.consumer.ConsumerConfig;
import kafka.consumer.ConsumerIterator;
import kafka.consumer.KafkaStream;
import kafka.javaapi.consumer.ConsumerConnector;
import backtype.storm.spout.SpoutOutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.topology.base.BaseRichSpout;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Values;
public class KafkaSpout extends BaseRichSpout {
private static final long serialVersionUID = -9174998944310422274L;
private SpoutOutputCollector collector;
private ConsumerConnector consumer;
private String topic;
public KafkaSpout() {}
public KafkaSpout(String topic) {
this.topic = topic;
}
public void nextTuple() { }
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
this.collector = collector;
}
public void ack(Object msgId) { }
public void activate() {
consumer =kafka.consumer.Consumer.createJavaConsumerConnector(createConsumerConfig());
Map<String,Integer> topickMap = new HashMap<String, Integer>();
topickMap.put(topic, 1);
System.out.println("*********Results********topic:"+topic);
Map<String, List<KafkaStream<byte[],byte[]>>> streamMap=consumer.createMessageStreams(topickMap);
KafkaStream<byte[],byte[]>stream = streamMap.get(topic).get(0);
ConsumerIterator<byte[],byte[]> it =stream.iterator();
while(it.hasNext()){
String value =new String(it.next().message());
System.out.println("storm接收到来自kafka的消息------->" + value);
collector.emit(new Values(value), value);
}
}
private static ConsumerConfig createConsumerConfig() {
Properties props = new Properties();
// 设置zookeeper的链接地址
props.put("zookeeper.connect","x01:2181,x02:2181");
// 设置group id
props.put("group.id", "1");
// kafka的group 消费记录是保存在zookeeper上的, 但这个信息在zookeeper上不是实时更新的, 需要有个间隔时间更新
props.put("auto.commit.interval.ms", "1000");
props.put("zookeeper.session.timeout.ms","10000");
return new ConsumerConfig(props);
}
public void close() { }
public void deactivate() { }
public void fail(Object msgId) { }
public void declareOutputFields(OutputFieldsDeclarer declarer) {
declarer.declare(new Fields("word"));
}
public Map<String, Object> getComponentConfiguration() {
System.out.println("getComponentConfiguration被调用");
topic="test";
return null;
}
}
bolt(wordsplitter)
package org.admln.flume_kafka_storm;
import java.util.Map;
import backtype.storm.task.OutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.topology.base.BaseRichBolt;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Tuple;
import backtype.storm.tuple.Values;
public class KafkaWordSplitterBolt extends BaseRichBolt {
private static final long serialVersionUID = 886149197481637894L;
private OutputCollector collector;
public void prepare(Map stormConf, TopologyContext context,
OutputCollector collector) {
this.collector = collector;
}
public void execute(Tuple input) {
String line = input.getString(0);
String[] words = line.split(",");
for(String word : words) {　　　　　　　　 //这里除了提交一个传向下个bolt的list集，还把tuple提交了，这是collector的emit方法之一，为了下面的ack错误恢复
collector.emit(input, new Values(word, 1));
}
collector.ack(input);
}
public void declareOutputFields(OutputFieldsDeclarer declarer) {
declarer.declare(new Fields("word", "count"));
}
}
bolt(wordcount)
package org.admln.flume_kafka_storm;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.atomic.AtomicInteger;
import backtype.storm.task.OutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.topology.base.BaseRichBolt;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Tuple;
public class KafkaWordCounterBolt extends BaseRichBolt {
private static final long serialVersionUID = 886149197481637894L;
private OutputCollector collector;
private Map<String, AtomicInteger> counterMap;
public void prepare(Map stormConf, TopologyContext context,
OutputCollector collector) {
this.collector = collector;
this.counterMap = new HashMap<String, AtomicInteger>();
}
public void execute(Tuple input) {
String word = input.getString(0);
int count = input.getInteger(1);
AtomicInteger ai = this.counterMap.get(word);
if (ai == null) {
ai = new AtomicInteger();
this.counterMap.put(word, ai);
}
ai.addAndGet(count);
collector.ack(input);
}
public void cleanup() {
Iterator<Entry<String, AtomicInteger>> iter = this.counterMap
.entrySet().iterator();
while (iter.hasNext()) {
Entry<String, AtomicInteger> entry = iter.next();
System.out.println(entry.getKey() + "\t:\t" + entry.getValue().get());
}
}
public void declareOutputFields(OutputFieldsDeclarer declarer) {
declarer.declare(new Fields("word", "count"));
}
}
topology
package org.admln.flume_kafka_storm;
import java.util.HashMap;
import java.util.Map;
import backtype.storm.Config;
import backtype.storm.LocalCluster;
import backtype.storm.StormSubmitter;
import backtype.storm.generated.AlreadyAliveException;
import backtype.storm.generated.InvalidTopologyException;
import backtype.storm.topology.TopologyBuilder;
import backtype.storm.tuple.Fields;
public class KafkaTopology {
public static void main(String[] args) throws AlreadyAliveException,
InvalidTopologyException {
TopologyBuilder builder = new TopologyBuilder();
builder.setSpout("spout", new KafkaSpout(""), 1);　　　　 //bolt1 是此bolt在这个图中的ID　　　　 //2表示启用多少线程来运行，可以省略，省略的话则默认分配一个线程
builder.setBolt("bolt1", new KafkaWordSplitterBolt(), 2)
.shuffleGrouping("spout");
builder.setBolt("bolt2", new KafkaWordCounterBolt(), 2).fieldsGrouping(
"bolt1", new Fields("word"));
String name = KafkaTopology.class.getSimpleName();
if (args != null && args.length > 0) {
Config conf = new Config();
// 通过指定nimbus主机
conf.put(Config.NIMBUS_HOST, args[0]);
conf.setNumWorkers(2);
StormSubmitter.submitTopologyWithProgressBar(name, conf,
builder.createTopology());
} else {
Map conf = new HashMap();
conf.put(Config.TOPOLOGY_WORKERS, 1);
conf.put(Config.TOPOLOGY_DEBUG, true);
LocalCluster cluster = new LocalCluster();
cluster.submitTopology("my-flume-kafka-storm-topology-integration",
conf, builder.createTopology());
}
}
}
可以直接在eclipse中本地运行也可以放到集群上运行
集群上
bin/storm jar flume-kafka-storm.jar org.admln.flume_kafka_storm.KafkaToplology x01

layne_liang

关注

0
点赞
踩
0

收藏

觉得还不错? 一键收藏
0
评论
flume+kafka+storm

centos06.6+JDK1.7flume1.4+kafka2.10+storm0.9.3zookeeper3.4.6集群：192.168.80.133 x01192.168.80.134 x021.两台机器上设置hostname和hosts。。。2.两台机器上安装JDK并设置环境变量3.下载安装zookeeper并设置环境变量
复制链接

扫一扫

专栏目录