1.flume配置
创建一个文件,flume监听这个文件,代替kafka生产者向消费者传输内容,具体配置如下
###############给sinnsource、channel起名字###############a
kafka_agent.sources = kafka_source
kafka_agent.sinks = kafka_sink
kafka_agent.channels = kafka_channel
##############配置source#####################
#对接文件
kafka_agent.sources.kafka_source.type = exec
kafka_agent.sources.kafka_source.command = tail -F /usr/local/flume1.8/test/MultilevelComputing.log
kafka_agent.sources.tailsource-1.shell = /bin/bash -c
###############配置sink######################
#对接kafka
kafka_agent.sinks.kafka_sink.type = org.apache.flume.sink.kafka.KafkaSink
#配置传输到哪个topic中
kafka_agent.sinks.kafka_sink.kafka.topic = Multilevel
#地址
kafka_agent.sinks.kafka_sink.kafka.bootstrap.servers = htkj101:9092
#批处理大小,1条消息处理一次
kafka_agent.sinks.kafka_sink.kafka.flumeBatchSize = 1
#输入数据传输成功策略
kafka_agent.sinks.kafka_sink.kafka.producer.acks = -1
kafka_agent.sinks.kafka_sink.kafka.producer.linger.ms = 1
kafka_agent.sinks.kafka_sink.kafka.producer.compression.type = snappy
############################配置channel###################
#对于channel的配置描述 使用文件做数据的临时缓存 这种的安全性要高
kafka_agent.channels.kafka_channel.type = file
kafka_agent.channels.kafka_channel.checkpointDir = /home/uplooking/data/flume/checkpoint
kafka_agent.channels.kafka_channel.dataDirs = /home/uplooking/data/flume/data
###########################整合三个组件#########################
kafka_agent.sources.kafka_source.channels = kafka_channel
kafka_agent.sinks.kafka_sink.channel = kafka_channel
2.storm接受kafka的数据并进行实时计算
2.1pom.xml
org.apache.storm
storm-core
1.1.0
org.slf4j
slf4j-log4j12
org.slf4j
log4j-over-slf4j
org.apache.kafka
kafka-clients
0.10.0.1
org.apache.kafka
kafka_2.11
0.10.0.1
org.apache.zookeeper
zookeeper
org.slf4j
slf4j-log4j12
org.slf4j
slf4j-api
org.slf4j
log4j-over-slf4j
org.apache.logging.log4j
log4j-slf4j-impl
org.apache.storm
storm-kafka-client
1.1.0
org.apache.storm
storm-jdbc
1.1.1
mysql
mysql-connector-java
5.1.31
2.2 KafkaSpout
从kafka中获取数据,同时也是topology,在这个类中进行配置
package com.htkj.multilevelcomputing.Storm;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.common.serialization.StringDeserializer;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
import org.apache.storm.kafka.spout.KafkaSpout;
import org.apache.storm.kafka.spout.KafkaSpoutConfig;
import org.apache.storm.topology.TopologyBuilder;
import org.apache.storm.tuple.Fields;
import org.apache.storm.utils.Utils;
public class KafkaTopo {
public static void main(String[] args) {
//创建TopologyBuilder
TopologyBuilder topologyBuilder =