Flume经典案例之自定义拦截器

需求分析：

对Source接收到的Event对象进行操作，Event对象：header和body。
body内容 以hello开头，则给event头header打入hello标签，body内容 以hi开头，则给event头header打入hi标签，其他的打上other标签。

操作步骤：

1、打开IDEA，新建maven工程。

2、导入依赖

<dependency>
      <groupId>org.apache.flume</groupId>
      <artifactId>flume-ng-core</artifactId>
      <version>1.6.0</version>
    </dependency>

3、编写代码

import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.interceptor.Interceptor;

import java.util.ArrayList;
import java.util.List;
import java.util.Map;

/**
 * 对Source接收到的
 * Event对象，：header，body
 * body内容 以hello开头，则给event头header打入hello标签
 * body内容 以hi开头，则给event头header打入hi标签
 */
public class InterceptorDemo implements Interceptor {
    ArrayList<Event> addHeaderEvents = null;

    @Override
    public void initialize() {
        addHeaderEvents = new ArrayList<>();
    }

    @Override
    public Event intercept(Event event) {
        Map<String, String> headers = event.getHeaders();

        byte[] body = event.getBody();
        String bodyStr = new String(body);
        if(bodyStr.startsWith("hello"))
            headers.put("type","hello");
        else if (bodyStr.startsWith("hi"))
            headers.put("type","hi");
        else
            headers.put("type","other");
        return event;
    }


    @Override
    public List<Event> intercept(List<Event> list) {
        for(Event event:list){
            Event opEvent = intercept(event);
            addHeaderEvents.add(opEvent);
        }
        return addHeaderEvents;
    }

    @Override
    public void close() {
        addHeaderEvents.clear();
        addHeaderEvents = null;
    }

    public static class Builder implements Interceptor.Builder{

        @Override
        public Interceptor build() {
            return new InterceptorDemo();
        }

        @Override
        public void configure(Context context) {
        }
    }
}

4、打包代码
使用xftp将包传入Linux中，传入flume目录下的lib目录中。

5、编写配置文件netcat-flume-interceptor.conf
内容如下：

interceptordemo.sources=interceptorDemoSource
interceptordemo.channels=helloChannel hiChannel otherChannel
interceptordemo.sinks=helloSink hiSink otherSink

interceptordemo.sources.interceptorDemoSource.type=netcat
interceptordemo.sources.interceptorDemoSource.bind=localhost
interceptordemo.sources.interceptorDemoSource.port=44444
interceptordemo.sources.interceptorDemoSource.interceptors=myinterceptors
interceptordemo.sources.interceptorDemoSource.interceptors.myinterceptors.type=cn.jit.InterceptorDemo$Builder
interceptordemo.sources.interceptorDemoSource.selector.type=multiplexing
interceptordemo.sources.interceptorDemoSource.selector.header=type
interceptordemo.sources.interceptorDemoSource.selector.mapping.hello=helloChannel
interceptordemo.sources.interceptorDemoSource.selector.mapping.hi=hiChannel
interceptordemo.sources.interceptorDemoSource.selector.mapping.other=otherChannel

interceptordemo.channels.helloChannel.type=memory
interceptordemo.channels.hiChannel.type=memory
interceptordemo.channels.otherChannel.type=memory

interceptordemo.sinks.helloSink.type=hdfs
interceptordemo.sinks.helloSink.hdfs.fileType=DataStream
interceptordemo.sinks.helloSink.hdfs.filePrefix=hello
interceptordemo.sinks.helloSink.hdfs.fileSuffix=.txt
interceptordemo.sinks.helloSink.hdfs.path=hdfs://192.168.91.135:9000/kb15file/hello/%Y-%m-%d
interceptordemo.sinks.helloSink.hdfs.useLocalTimeStamp=true
interceptordemo.sinks.helloSink.hdfs.batchSize=640
interceptordemo.sinks.helloSink.hdfs.rollCount=0
interceptordemo.sinks.helloSink.hdfs.rollSize=64000000
interceptordemo.sinks.helloSink.hdfs.rollInterval=3

interceptordemo.sinks.hiSink.type=hdfs
interceptordemo.sinks.hiSink.hdfs.fileType=DataStream
interceptordemo.sinks.hiSink.hdfs.filePrefix=hi
interceptordemo.sinks.hiSink.hdfs.fileSuffix=.txt
interceptordemo.sinks.hiSink.hdfs.path=hdfs://192.168.91.135:9000/kb15file/hi/%Y-%m-%d
interceptordemo.sinks.hiSink.hdfs.useLocalTimeStamp=true
interceptordemo.sinks.hiSink.hdfs.batchSize=640
interceptordemo.sinks.hiSink.hdfs.rollCount=0
interceptordemo.sinks.hiSink.hdfs.rollSize=64000000
interceptordemo.sinks.hiSink.hdfs.rollInterval=3

interceptordemo.sinks.otherSink.type=logger

interceptordemo.sources.interceptorDemoSource.channels=helloChannel hiChannel otherChannel
interceptordemo.sinks.helloSink.channel=helloChannel
interceptordemo.sinks.hiSink.channel=hiChannel
interceptordemo.sinks.otherSink.channel=otherChannel

6、执行命令
进入到flume目录下，执行命令./bin/flume-ng agent --name interceptordemo --conf conf/ --conf-file conf/KB15conf/netcat-flume-interceptor.conf -Dflume.root.logger=INFO,console

7、查看输出内容

因为other是以logger的形式输出，所以并没有生成文件。