一. maven pom.xml
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.cn.bigdata</groupId>
<artifactId>flume-interceptor</artifactId>
<version>1.0-SNAPSHOT</version>
<name>flume-interceptor</name>
<description>flume-interceptor</description>
<packaging>jar</packaging>
<properties>
<java.version>1.8</java.version>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<slf4j.version>1.7.21</slf4j.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.flume</groupId>
<artifactId>flume-ng-core</artifactId>
<version>1.9.0</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.62</version>
</dependency>
</dependencies>
<build>
<finalName>flume-interceptor-1.0.0</finalName>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.5.1</version>
<configuration>
<source>${maven.compiler.source}</source>
<target>${maven.compiler.target}</target>
<compilerArgument>-parameters</compilerArgument>
</configuration>
</plugin>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
<distributionManagement>
<repository>
<id>nexus-releases</id>
<name>maven-releases</name>
<url>http://xx.xx.xx.xx:8081/nexus/content/repositories/snapshots/</url>
</repository>
</distributionManagement>
</project>
二. Flume拦截器
package com.cn.bigdata.flume;
import com.alibaba.fastjson.JSON;
import com.google.common.base.Charsets;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.cn.bigdata.flume.model.BinlogSyncDto;
import com.cn.bigdata.flume.model.BinlogType;
import org.apache.commons.lang.StringUtils;
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.interceptor.Interceptor;
import org.joda.time.LocalDate;
import java.util.Date;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
/**
* @description 拦截器
*/
public class FlumeInterceptor implements Interceptor {
private static final String UNKNOW_MARK = "unknow";
private static final List<String> ALLOW_EVENT_TYPES = Lists.newArrayList(
BinlogType.DELETE.name(),
BinlogType.INSERT.name(),
BinlogType.UPDATE.name());
@Override
public void close() {
}
@Override
public void initialize() {
}
@Override
public Event intercept(Event event) {
try {
String body = new String(event.getBody(), Charsets.UTF_8);
if (StringUtils.isBlank(body)) {
return null;
}
BinlogSyncDto binlogSyncDto = JSON.parseObject(body,BinlogSyncDto.class);
if (!ALLOW_EVENT_TYPES.contains(binlogSyncDto.getType())) {
return null;
}
event.getHeaders().put("eventDate",LocalDate.fromDateFields(new Date(binlogSyncDto.getEs())).toString());
event.setBody(JSON.toJSONString(binlogSyncDto).getBytes());
}catch (Exception e){
//脏数据,需要sink到一个目录进行核查
event.getHeaders().put("eventDate", UNKNOW_MARK);
event.setBody(unknowBody(e.getMessage()).getBytes());
}
return event;
}
private String unknowBody(String errMsg){
BinlogSyncDto binlogSyncDto = new BinlogSyncDto();
binlogSyncDto.setEs(System.currentTimeMillis());
binlogSyncDto.setType("ERROR");
binlogSyncDto.setData(Lists.newArrayList(ImmutableMap.of("status","500","msg",errMsg)));
return JSON.toJSONString(binlogSyncDto);
}
@Override
public List<Event> intercept(List<Event> events) {
List<Event> eventList = Lists.newArrayList();
for(final Event event : events) {
eventList.add(intercept(event));
}
return eventList.stream().filter(Objects::nonNull).collect(Collectors.toList());
}
public static class Builder implements Interceptor.Builder{
@Override
public void configure(Context context) {
}
@Override
public Interceptor build() {
return new FlumeInterceptor();
}
}
}
package com.cn.bigdata.flume.model;
public enum BinlogType {
DELETE,
UPDATE,
INSERT;
private BinlogType() {
}
}
package com.cn.bigdata.flume.model;
import java.util.List;
import java.util.Map;
public class BinlogSyncDto {
private List<Map<String, Object>> data;
private List<Map<String, Object>> old;
private String database;
private long es;
private String table;
private String type;
public BinlogSyncDto() {
}
public List<Map<String, Object>> getData() {
return this.data;
}
public void setData(List<Map<String, Object>> data) {
this.data = data;
}
public List<Map<String, Object>> getOld() {
return this.old;
}
public void setOld(List<Map<String, Object>> old) {
this.old = old;
}
public String getDatabase() {
return this.database;
}
public void setDatabase(String database) {
this.database = database;
}
public long getEs() {
return this.es;
}
public void setEs(long es) {
this.es = es;
}
public String getTable() {
return this.table;
}
public void setTable(String table) {
this.table = table;
}
public String getType() {
return this.type;
}
public void setType(String type) {
this.type = type;
}
}