Flume拦截器

一. maven pom.xml

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>


    <groupId>com.cn.bigdata</groupId>
    <artifactId>flume-interceptor</artifactId>
    <version>1.0-SNAPSHOT</version>
    <name>flume-interceptor</name>
    <description>flume-interceptor</description>
    <packaging>jar</packaging>

    <properties>
        <java.version>1.8</java.version>
        <maven.compiler.source>1.8</maven.compiler.source>
        <maven.compiler.target>1.8</maven.compiler.target>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
        <slf4j.version>1.7.21</slf4j.version>

    </properties>

    <dependencies>
        <dependency>
            <groupId>org.apache.flume</groupId>
            <artifactId>flume-ng-core</artifactId>
            <version>1.9.0</version>
        </dependency>
        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>fastjson</artifactId>
            <version>1.2.62</version>
        </dependency>
    </dependencies>

    <build>
        <finalName>flume-interceptor-1.0.0</finalName>
        <plugins>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-compiler-plugin</artifactId>
                <version>3.5.1</version>
                <configuration>
                    <source>${maven.compiler.source}</source>
                    <target>${maven.compiler.target}</target>
                    <compilerArgument>-parameters</compilerArgument>
                </configuration>
            </plugin>
            <plugin>
                <artifactId>maven-assembly-plugin</artifactId>
                <configuration>
                    <descriptorRefs>
                        <descriptorRef>jar-with-dependencies</descriptorRef>
                    </descriptorRefs>
                </configuration>
                <executions>
                    <execution>
                        <id>make-assembly</id>
                        <phase>package</phase>
                        <goals>
                            <goal>single</goal>
                        </goals>
                    </execution>
                </executions>
            </plugin>
        </plugins>
    </build>

    <distributionManagement>
        <repository>
            <id>nexus-releases</id>
            <name>maven-releases</name>
            <url>http://xx.xx.xx.xx:8081/nexus/content/repositories/snapshots/</url>
        </repository>
    </distributionManagement>

</project>

二. Flume拦截器

package com.cn.bigdata.flume;
import com.alibaba.fastjson.JSON;
import com.google.common.base.Charsets;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.cn.bigdata.flume.model.BinlogSyncDto;
import com.cn.bigdata.flume.model.BinlogType;
import org.apache.commons.lang.StringUtils;
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.interceptor.Interceptor;
import org.joda.time.LocalDate;

import java.util.Date;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;

/**
 * @description 拦截器
 */

public class FlumeInterceptor implements Interceptor {

    private static final String UNKNOW_MARK = "unknow";
    private static final List<String> ALLOW_EVENT_TYPES = Lists.newArrayList(
            BinlogType.DELETE.name(),
            BinlogType.INSERT.name(),
            BinlogType.UPDATE.name());

    @Override
    public void close() {

    }

    @Override
    public void initialize() {
    }

    @Override
    public Event intercept(Event event) {
        try {
            String body = new String(event.getBody(), Charsets.UTF_8);
            if (StringUtils.isBlank(body)) {
                return null;
            }
            BinlogSyncDto binlogSyncDto = JSON.parseObject(body,BinlogSyncDto.class);
            if (!ALLOW_EVENT_TYPES.contains(binlogSyncDto.getType())) {
                return null;
            }
            event.getHeaders().put("eventDate",LocalDate.fromDateFields(new Date(binlogSyncDto.getEs())).toString());
            event.setBody(JSON.toJSONString(binlogSyncDto).getBytes());
        }catch (Exception e){
            //脏数据,需要sink到一个目录进行核查
            event.getHeaders().put("eventDate", UNKNOW_MARK);
            event.setBody(unknowBody(e.getMessage()).getBytes());
        }
        return event;
    }

    private String unknowBody(String errMsg){
        BinlogSyncDto binlogSyncDto = new BinlogSyncDto();
        binlogSyncDto.setEs(System.currentTimeMillis());
        binlogSyncDto.setType("ERROR");
        binlogSyncDto.setData(Lists.newArrayList(ImmutableMap.of("status","500","msg",errMsg)));
        return JSON.toJSONString(binlogSyncDto);
    }

    @Override
    public List<Event> intercept(List<Event> events) {
        List<Event> eventList = Lists.newArrayList();
        for(final Event event : events) {
            eventList.add(intercept(event));
        }
        return eventList.stream().filter(Objects::nonNull).collect(Collectors.toList());
    }

    public static class Builder implements Interceptor.Builder{

        @Override
        public void configure(Context context) {
        }

        @Override
        public Interceptor build() {
            return new FlumeInterceptor();
        }
    }

}

package com.cn.bigdata.flume.model;
public enum BinlogType {
    DELETE,
    UPDATE,
    INSERT;

    private BinlogType() {
    }
}

package com.cn.bigdata.flume.model;
import java.util.List;
import java.util.Map;
public class BinlogSyncDto {

    private List<Map<String, Object>> data;
    private List<Map<String, Object>> old;
    private String database;
    private long es;
    private String table;
    private String type;

    public BinlogSyncDto() {
    }

    public List<Map<String, Object>> getData() {
        return this.data;
    }

    public void setData(List<Map<String, Object>> data) {
        this.data = data;
    }

    public List<Map<String, Object>> getOld() {
        return this.old;
    }

    public void setOld(List<Map<String, Object>> old) {
        this.old = old;
    }

    public String getDatabase() {
        return this.database;
    }

    public void setDatabase(String database) {
        this.database = database;
    }

    public long getEs() {
        return this.es;
    }

    public void setEs(long es) {
        this.es = es;
    }

    public String getTable() {
        return this.table;
    }

    public void setTable(String table) {
        this.table = table;
    }

    public String getType() {
        return this.type;
    }

    public void setType(String type) {
        this.type = type;
    }
}