flume对接kafka

需求 flume采集的数据发送到不同的kafka topic里面。
flume-kafka.conf

define

a1.sources = r1
a1.sinks = k1
a1.channels = c1

source

a1.sources.r1.type = exec
a1.sources.r1.command = tail -F -c +0 /opt/module/datas/flume.log
a1.sources.r1.shell = /bin/bash -c

sink

a1.sinks.k1.type = org.apache.flume.sink.kafka.KafkaSink
a1.sinks.k1.kafka.bootstrap.servers = hadoop203:9092,hadoop204:9092,hadoop205:9092
a1.sinks.k1.kafka.topic = first
a1.sinks.k1.kafka.flumeBatchSize = 20
a1.sinks.k1.kafka.producer.acks = 1
a1.sinks.k1.kafka.producer.linger.ms = 1

channel

a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100

bind

a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1

2.启动kafka消费者
kafka-console-consumer.sh --zookeeper hadoop203:2181 --topic first
3.启动flume
bin/flume-ng agent -c conf/ -f job/flume-kafka.conf -n a1
4. nc localhost 44444
5.经测试 比单纯的flume对接flume速度要快
信息分类
flume利用拦截器把信息分类 结合channel选择器发送到不同的channel
如果用kafkasink更简单 aaa first
1.编写Flume拦截器TypeInterceptor
2.public class TypeInterceptor implements Interceptor {
private List addHeaderEverts; //定义添加过头的eventlist；
//初始化方法
@Override
public void initialize() {
addHeaderEverts = new ArrayList();
}
//单个事件拦截
@Override
public Event intercept(Event event) {
//获取事件的头信息
Map<String, String> headers = event.getHeaders();
//获取事件中的body信息
byte[] bytes = event.getBody();
String body = new String(bytes);
//根据body中是否含有“hello”来决定添加怎么样的头信息
if(body.contains(“hello”)){
headers.put(“topic”, “first”);//含有hello的添加<type,atguigu>
}else {
headers.put(“topic”, “aaa”);//不含有hello的添加<type,bigdata>
}
return event;
}
//批量事件拦截
@Override
public List intercept(List events) {
//1.先清空集合
addHeaderEverts.clear();
//2.遍历事件，
for (Event event : events) {
//3.给每个事件添加头信息
addHeaderEverts.add(intercept(event));
}

    //4.返回list.
    return addHeaderEverts;
}
@Override
public void close() {

}
//自定义拦截器最容易遗漏的
public static class Builer implements Interceptor.Builder{
    @Override
    public Interceptor build() {
        return new TypeInterceptor();
    }

    @Override
    public void configure(Context context) {

    }
}

}
定义flume配置文件 不同的信息发送到不同的kafka主题。
#Name
a1.sources = r1
a1.channels = c1
a1.sinks = k1
#Soucre
a1.sources.r1.type = netcat
a1.sources.r1.bind = localhost
a1.sources.r1.port = 44444
#Interceptor
a1.sources.r1.interceptors = i1
a1.sources.r1.interceptors.i1.type = com.atguigu.interceptor.TypeInterceptor$Builer
#channel
a1.channels.c1.type=memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100
#Sink
a1.sinks.k1.type = org.apache.flume.sink.kafka.KafkaSink
a1.sinks.k1.kafka.bootstrap.servers = hadoop203:9092,hadoop204:9092,hadoop205:9092
a1.sinks.k1.kafka.topic = first
a1.sinks.k1.kafka.flumeBatchSize = 20
a1.sinks.k1.kafka.producer.acks = 1
a1.sinks.k1.kafka.producer.linger.ms = 1

bind

a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1
kafka-console-consumer.sh --zookeeper hadoop203:2181 --topic first
kafka-console-consumer.sh --zookeeper hadoop203:2181 --topic aaa