Flume自定义拦截器-筛选出以某文本开头的内容

简介

• 本文拦截器主要是将以时间开头包含推荐用户行为信息：的日志数据筛选出来，筛选的结果存到kafka的topic中
• 测试数据 ：2022-16-17 13:52:30 推荐用户行为信息：你好，BIGDATA_LYT
• 结果数据：你好，BIGDATA_LYT

架构图

pom文件加载依赖

<dependencies>
        <dependency>
            <groupId>org.apache.flume</groupId>
            <artifactId>flume-ng-core</artifactId>
            <version>1.9.0</version>
            <scope>provided</scope>
        </dependency>

        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>fastjson</artifactId>
            <version>1.2.62</version>
        </dependency>
    </dependencies>

    <build>
        <plugins>
            <plugin>
                <artifactId>maven-compiler-plugin</artifactId>
                <version>2.3.2</version>
                <configuration>
                    <source>1.8</source>
                    <target>1.8</target>
                </configuration>
            </plugin>

            <plugin>
                <artifactId>maven-assembly-plugin</artifactId>
                <configuration>
                    <descriptorRefs>
                        <descriptorRef>jar-with-dependencies</descriptorRef>
                    </descriptorRefs>
                </configuration>
                <executions>
                    <execution>
                        <id>make-assembly</id>
                        <phase>package</phase>
                        <goals>
                            <goal>single</goal>
                        </goals>
                    </execution>
                </executions>
            </plugin>
        </plugins>
    </build>

自定义拦截器代码

package com.lyt.gmall.flume.interceptor;


import com.google.common.base.Charsets;
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.interceptor.Interceptor;

import java.nio.charset.StandardCharsets;
import java.util.Iterator;
import java.util.List;


public class ETLInterceptor implements Interceptor {

    @Override
    public void initialize() {

    }

    @Override
    public Event intercept(Event event) {
        //1、获取Body中的数据
        byte[] body = event.getBody();
        String log = new String(body, StandardCharsets.UTF_8);

        //2、筛选出以时间开头，包含推荐用户行为信息：的内容
        if (log.matches("^2.*推荐用户行为信息：.*")) {
            //3、将以时间开头，"推荐用户行为信息："结尾的替换成空
            log=log.replaceAll("^2.*推荐用户行为信息：", "");
            //4、将event的内容替换成log
            event.setBody(log.getBytes(Charsets.UTF_8));
        } else {
            //5、如果不包含"以时间开头，推荐用户行为信息：结尾"，将信息置为空值
            log = null;
        }
        //6、如果log不为空值，返回event
        if (log != null) {
            return event;
        } else {
            //7、如果log为空值，返回null
            return null;
        }

    }

    @Override
    public List<Event> intercept(List<Event> list) {
        //8、将list列表转换为迭代器
        Iterator<Event> iterator = list.iterator();
        //9、使用while循环，判断迭代器是否有下一个值
        while (iterator.hasNext()) {
            //10、获取迭代器下一个值
            Event event = iterator.next();
            //11、如果事件值为空，将事件从迭代器中删除
            if (intercept(event) == null) {
                iterator.remove();
            }
        }
        //12、返回处理好的列表
        return list;
    }

    @Override
    public void close() {

    }

    public static class Builder implements Interceptor.Builder {

        @Override
        public Interceptor build() {
            //13、new一个ETL拦截器对象
            return new ETLInterceptor();
        }

        @Override
        public void configure(Context context) {

        }
    }
}

flume配置文件

a1.sources = r1
a1.channels = c1

#define sources
a1.sources.r1.type = TAILDIR
a1.sources.r1.filegroups = f1
a1.sources.r1.filegroups.f1 = /opt/module/2.log
a1.sources.r1.positionFile.f1 = /opt/soft/flume/tail.json


a1.sources.r1.interceptors = i1
a1.sources.r1.interceptors.i1.type = com.lyt.gmall.flume.interceptor.ETLInterceptor$Builder

#define channels
a1.channels.c1.type = org.apache.flume.channel.kafka.KafkaChannel
a1.channels.c1.kafka.bootstrap.servers = hadoop101:9092,hadoop102:9092,hadoop103:9092
a1.channels.c1.kafka.topic = topic_log
a1.channels.c1.parseAsFlumeEvent = false

#bind sources and sink to channel 
a1.sources.r1.channels = c1

flume启动命令

bin/flume-ng agent -n a1 -c conf/ -f job/log_to_kafka.conf -Dflume.root.logger=info,console