Flume拦截器

LogETLInterceptor

package com.atguigu.flume.interceptor;

import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.interceptor.Interceptor;

import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;

/**
 * @author Thomas
 * @version 1.0
 * @date 2020/5/12 7:21
 */
public class LogETLInterceptor implements Interceptor {
    @Override
    public void initialize() {

    }

    @Override
    public Event intercept(Event event) {

        //清洗数据 ETL

        //1.获取日志
        byte[] body = event.getBody();
        String log = new String(body, Charset.forName("UTF-8"));

        //2.区分数据
        if (log.contains("start")){
            //验证启动日志
            if(LogUtils.validateStart(log)){
                return event;
            }
        }else {
            //验证事务日志
            if(LogUtils.validateEvent(log)){
                return event;
            }
        }

        return null;
    }

    @Override
    public List<Event> intercept(List<Event> events) {

        //多Event处理
        ArrayList<Event> intercepts = new ArrayList<>();

        for (Event event : events) {
            Event intercept1 = intercept(event);

            if (intercept1 != null){
                intercepts.add(intercept1);
            }
        }

        return intercepts;
    }

    @Override
    public void close() {

    }

    public static class Builder implements Interceptor.Builder{

        @Override
        public Interceptor build() {
            return new LogETLInterceptor();
        }

        @Override
        public void configure(Context context) {

        }
    }
}

LogTypeInterceptor

package com.atguigu.flume.interceptor;

import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.interceptor.Interceptor;

import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;

/**
 * @author Thomas
 * @version 1.0
 * @date 2020/5/12 7:51
 */
public class LogTypeInterceptor implements Interceptor {
    @Override
    public void initialize() {

    }

    @Override
    public Event intercept(Event event) {

        //区分类型 start event
        //body header
        //获取body
        byte[] body = event.getBody();
        String log = new String(body, Charset.forName("UTF-8"));

        //获取头信息
        Map<String, String> headers = event.getHeaders();

        if (log.contains("start")){
            headers.put("topic", "topic_start");
        }else {
            headers.put("topic", "topic_event");
        }

        return event;
    }

    @Override
    public List<Event> intercept(List<Event> events) {

        ArrayList<Event> intercepts = new ArrayList<>();

        for (Event event : events) {
            Event intercept1 = intercept(event);

            intercepts.add(intercept1);
        }

        return intercepts;
    }

    @Override
    public void close() {

    }

    public static class Builder implements Interceptor.Builder{

        @Override
        public Interceptor build() {
            return new LogTypeInterceptor();
        }

        @Override
        public void configure(Context context) {

        }
    }
}

LogUtils

package com.atguigu.flume.interceptor;

import org.apache.commons.lang.math.NumberUtils;

/**
 * @author Thomas
 * @version 1.0
 * @date 2020/5/12 7:32
 */
public class LogUtils {

    //验证启动日志
    public static boolean validateStart(String log) {

        if (log == null){
            return false;
        }

        //判断数据是否合法
        if (!log.trim().startsWith("{") || !log.trim().endsWith("}")){
            return false;
        }

        return true;
    }

    //验证事务日志
    public static boolean validateEvent(String log) {

        if (log == null){
            return false;
        }

        String[] logContents = log.split("\\|");

        if (logContents.length != 2){
            return false;
        }

        if (logContents[0].length() != 13 || !NumberUtils.isDigits(logContents[0])){
            return false;
        }

        if (!logContents[1].trim().startsWith("{") || !logContents[1].trim().endsWith("}")){
            return false;
        }

        return true;
    }
}

pom.xml

   <dependencies>
        <dependency>
            <groupId>org.apache.flume</groupId>
            <artifactId>flume-ng-core</artifactId>
            <version>1.7.0</version>
        </dependency>
    </dependencies>

    <build>
        <plugins>
            <plugin>
                <artifactId>maven-compiler-plugin</artifactId>
                <version>2.3.2</version>
                <configuration>
                    <source>1.8</source>
                    <target>1.8</target>
                </configuration>
            </plugin>
            <plugin>
                <artifactId>maven-assembly-plugin</artifactId>
                <configuration>
                    <descriptorRefs>
                        <descriptorRef>jar-with-dependencies</descriptorRef>
                    </descriptorRefs>
                </configuration>
                <executions>
                    <execution>
                        <id>make-assembly</id>
                        <phase>package</phase>
                        <goals>
                            <goal>single</goal>
                        </goals>
                    </execution>
                </executions>
            </plugin>
        </plugins>
    </build>

打包上传到Flume的lib包中