flume-自定义拦截器interceptor

一、先在idea里面创建好自定义拦截器函数

1、配置pom.xml

<!--flume核心组件-->
    <dependency>
      <groupId>org.apache.flume</groupId>
      <artifactId>flume-ng-core</artifactId>
      <version>1.6.0</version>
    </dependency>

2、编写idea 代码

package nj.zb;

import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.interceptor.Interceptor;

import java.util.List;
import java.util.Map;

/**
 * @author: 03-CJH
 * @date:2020/5/23
 * @desc: flume自定义代码部分
 */
public class InterceptorDemo implements Interceptor {
    private List<Event> opList;
    @Override
    public void initialize() {

    }

    @Override
    public Event intercept(Event event) {
        Map<String, String> headers = event.getHeaders();
        String body = new String(event.getBody());
        if (body.startsWith("hello")){
            headers.put("type","hello");
        }else {
            headers.put("type","other");
        }
        return event;
    }
    /**
     *这里编写对于event对象集合的处理代码，一般都是遍历event的对象集合，对于每一个		                   event对象，调用 Event intercept(Event event) 方法，然后根据返回值是否为null，来将其添加    到新的集合中。 
     */
    @Override
    public List<Event> intercept(List<Event> list) {
        opList.clear();
        for (Event event : list) {
            opList.add(intercept(event));
        }
        return opList;
    }
    
    /**
     * 该方法主要用来销毁拦截器对象值执行，一般是一些释放资源的处理
     */
    @Override
    public void close() {

    }
	// 自定义拦截器内部添加静态内部类
	/**
     * 通过该静态内部类来创建自定义对象供flume使用，实现Interceptor.Builder接口，并实现  其抽象方法
     */
    public static class Builder implements Interceptor.Builder{
   	   /**
         * 该方法主要用来返回创建的自定义类拦截器对象
         */
        @Override
        public Interceptor build() {
            return new InterceptorDemo();
        }
        @Override
        public void configure(Context context) {

        }
    }

}

3、打包测试

然后将打包的jar包放到flume的lib目录下

二、打开虚拟机，在Xshell编写自定义拦截器文件

1、编写 flume 配置文件

# 创建文件
[root@cjh1 job]# touch netcat-flume-interceptor-hdfs.conf 
[root@cjh1 job]# vi netcat-flume-interceptor-hdfs.conf 

# Name the components on this agent, a1:表示agent的名称
a1.sources = r1
a1.channels = c1 c2
a1.sinks = k1 k2

# Describe/configure the source
a1.sources.r1.type = netcat
a1.sources.r1.bind = localhost
a1.sources.r1.port = 44444
a1.sources.r1.interceptors = i1
a1.sources.r1.interceptors.i1.type = nj.zb.InterceptorDemo$Builder
a1.sources.r1.selector.type = multiplexing
a1.sources.r1.selector.header = type
a1.sources.r1.selector.mapping.hello = c1
a1.sources.r1.selector.mapping.other = c2

# Describe the sink
a1.sinks.k1.type = hdfs
a1.sinks.k1.hdfs.fileType = DataStream
a1.sinks.k1.hdfs.filePrefix = hello
a1.sinks.k1.hdfs.fileSuffix = .csv
a1.sinks.k1.hdfs.path = hdfs://192.168.48.141:9000/user/hello/%Y-%m-%d
a1.sinks.k1.hdfs.useLocalTimeStamp = true
a1.sinks.k1.hdfs.batchSize = 640
a1.sinks.k1.hdfs.rollCount = 0
a1.sinks.k1.hdfs.rollSize = 1000
a1.sinks.k1.hdfs.rollInterval = 3

a1.sinks.k2.type = hdfs
# 设置文件类型，可支持压缩
a1.sinks.k2.hdfs.fileType = DataStream
# 上传文件的前缀
a1.sinks.k2.hdfs.filePrefix = other   
a1.sinks.k2.hdfs.fileSuffix = .csv
a1.sinks.k2.hdfs.path = hdfs://192.168.48.141:9000/user/other/%Y-%m-%d
# 是否使用本地时间戳
a1.sinks.k2.hdfs.useLocalTimeStamp = true
# 积攒多少个 Event 才 flush 到 HDFS 一次
a1.sinks.k2.hdfs.batchSize = 640
# 文件的滚动与 Event 数量无关
a1.sinks.k2.hdfs.rollCount = 0
# 设置每个文件的滚动大小
a1.sinks.k2.hdfs.rollSize = 1000
# 多久生成一个新的文件
a1.sinks.k2.hdfs.rollInterval = 3

# Use a channel which buffers events in memory
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100

a1.channels.c2.type = memory
a1.channels.c2.capacity = 1000
a1.channels.c2.transactionCapacity = 100

# Bind the source and sink to the channel
a1.sources.r1.channels = c1 c2
a1.sinks.k1.channel = c1
a1.sinks.k2.channel = c2

2、启动flume前需启动hadoop

# 启动hadoop
[root@cjh1 flume160]# start-all.sh

# 启动flume-agent服务
[root@cjh1 flume160]# ./bin/flume-ng agent -c conf/ -f conf/job/netcat-flume-interceptor-hdfs.conf -n a1 -Dflume.root.logger=INFO,console

3、然后打开一个新窗口测试执行

# 向监控端口发送数据
[root@cjh1 checkpoint]# nc localhost 44444
hello
OK
cjh
OK

输入hello 和 cjh 就在hdfs下生成了两个不一样的新文件

4、在网页下查看或者hdfs下查看两个新文件

[root@cjh1 ~]# hdfs dfs -cat /user/hello/2020-05-21/hello.1590033842239.csv

20/05/21 12:05:43 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
hello

[root@cjh1 ~]# hdfs dfs -cat /user/other/2020-05-21/other.1590033850233.csv

20/05/21 12:06:50 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
cjh

自此，达到了flume自定义拦截器的目的.
ps:望多多支持，在下必再接再厉！！！