Flume(8) flume自定义Interceptors实现

源码地址

接上两篇Flume(6) flume自定义Sources实现 Flume(7) flume自定义Sinks实现，我们总结了一下自定义source和sink的流程，这次我们实现一个自己的拦截器实现，用来数据清洗和过滤

自定义拦截器实现说明

1. 实现interceptor接口，并实现其方法，接口完全限定名为：org.apache.flume.interceptor.Interceptor;

2. 自定义拦截器内部添加静态内部类，实现Builder接口，并实现其方法，接口完全限定名为：Interceptor.Builder

添加依赖

<properties>
    <version.flume>1.9.0</version.flume>
</properties>

<dependencies>
<dependency>
    <groupId>org.apache.flume</groupId>
        <artifactId>flume-ng-core</artifactId>
        <version>${version.flume}</version>
    </dependency>
    <dependency>
    <groupId>org.apache.flume</groupId>
        <artifactId>flume-ng-configuration</artifactId>
        <version>${version.flume}</version>
    </dependency>
</dependencies>

编写拦截器

public class MyInterceptor implements Interceptor,Interceptor.Builder {
    private static final Logger logger = LoggerFactory.getLogger(MyInterceptor.class);
    //拦截字符串
    private String filterStr;
    @Override
    public void initialize() {

    }

    @Override
    public Event intercept(Event event) {
        try {
            String body = new String(event.getBody(), "UTF-8");
            logger.info("filter:{}",body);
            if(body!=null){
                //过滤以a开始的字符串
                if(body.contains(filterStr)){
                    logger.info("{}被过滤........",body);
                    return null;
                }
                body = body + ":intercept";
                return EventBuilder.withBody(body.getBytes());
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
        return null;
    }

    @Override
    public List<Event> intercept(List<Event> events) {
        logger.info("filter:{}","list intercept");
        return null;
    }

    @Override
    public void close() {
    }

    @Override
    public Interceptor build() {
        return this;
    }

    @Override
    public void configure(Context context) {
        filterStr = context.getString("filter_str","a");
    }

}

实现对特定字符串的过滤

编写filter.conf

#定义agent名， source、channel、sink的名称
a1.sources = r1
#定义agent名， source、channel、sink的名称
a1.sources = r1
a1.channels = c1
a1.sinks = k1

#具体定义source,这里的type是自定义的source的类的全路径
a1.sources.r1.type = cn.myframe.source.TailFileSource
#这里的参数名都和自定义类的参数一直
#读取哪个文件
a1.sources.r1.filePath = /usr/local/flume/data2/a.txt
#偏移量保存的文件
a1.sources.r1.positionFile = /usr/local/flume/data2/index
#时间间隔，每隔多久读取一次
a1.sources.r1.interval = 5000
#编码
a1.sources.r1.charset = UTF-8

#具体定义channel
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100

#具体定义sink
a1.sinks.k1.type = cn.myframe.sink.MysqlSink
a1.sinks.k1.column_name = content
a1.sinks.k1.tableName= flume
a1.sinks.k1.url= jdbc:mysql://10.10.2.137:3306/flume?allowMultiQueries=true&useUnicode=true&characterEncoding=UTF-8
a1.sinks.k1.user= root
a1.sinks.k1.password= Rojao@123


#组装source、channel、sink
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1

a1.sources.r1.interceptors=i1
a1.sources.r1.interceptors.i1.type=cn.myframe.interceptor.MyInterceptor
a1.sources.r1.interceptors.i1.filter_str = abc

启动

flume-ng agent --conf /usr/local/flume/apache-flume-1.9.0-bin/conf --conf-file  /usr/local/flume/apache-flume-1.9.0-bin/conf/filter.conf --name a1 -Dflume.root.logger=INFO,console