一 、拦截器
- 拦截器接口
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.flume.interceptor;
import java.util.List;
import org.apache.flume.Event;
import org.apache.flume.annotations.InterfaceAudience;
import org.apache.flume.annotations.InterfaceStability;
import org.apache.flume.conf.Configurable;
@InterfaceAudience.Public
@InterfaceStability.Stable
public interface Interceptor {
/**
* Any initialization / startup needed by the Interceptor.
* 在启动拦截器时做的一些初始化操作
*/
public void initialize();
/**
* Interception of a single {@link Event}.
* @param event Event to be intercepted
* @return Original or modified event, or {@code null} if the Event
* is to be dropped (i.e. filtered out).
* 每次取一个事件进行事务操作
*/
public Event intercept(Event event);
/**
* Interception of a batch of {@linkplain Event events}.
* @param events Input list of events
* @return Output list of events. The size of output list MUST NOT BE GREATER
* than the size of the input list (i.e. transformation and removal ONLY).
* Also, this method MUST NOT return {@code null}. If all events are dropped,
* then an empty List is returned.
* 每次取一个集合的事务操作
*/
public List<Event> intercept(List<Event> events);
/**
* Perform any closing / shutdown needed by the Interceptor.
* 主要是对在初始化时的一些需要关闭的操作
*/
public void close();
/** Builder implementations MUST have a no-arg constructor
* 对拦截器内部功能需要用到的一些参数的配置
*/
public interface Builder extends Configurable {
public Interceptor build();
}
}
通过上面的接口文档可以知道拦截器的实现很简单,实现对应的方法就可以了。注意拦截器的处理逻辑必须确保是线程安全的。
-
时间戳拦截器源码分析
时间戳拦截器是官方提供的源码通过它来加深对拦截器的理解消化。
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.flume.interceptor;
import java.util.List;
import java.util.Map;
import org.apache.flume.Context;
import org.apache.flume.Event;
import static org.apache.flume.interceptor.TimestampInterceptor.Constants.*;
/**
* Simple Interceptor class that sets the current system timestamp on all events
* that are intercepted.
* By convention, this timestamp header is named "timestamp" by default and its format
* is a "stringified" long timestamp in milliseconds since the UNIX epoch.
* The name of the header can be changed through the configuration using the
* config key "header".
*/
public class TimestampInterceptor implements Interceptor {
private final boolean preserveExisting;
private final String header;
/**
* Only {@link TimestampInterceptor.Builder} can build me
*/
private TimestampInterceptor(boolean preserveExisting, String header) {
this.preserveExisting = preserveExisting;
this.header = header;
}
@Override
public void initialize() {
// no-op
}
/**
* Modifies events in-place.
*/
@Override
public Event intercept(Event event) {
Map<String, String> headers = event.getHeaders();
if (preserveExisting && headers.containsKey(header)) {
// we must preserve the existing timestamp
} else {
long now = System.currentTimeMillis();
headers.put(header, Long.toString(now));
}
return event;
}
/**
* Delegates to {@link #intercept(Event)} in a loop.
* @param events
* @return
*/
@Override
public List<Event> intercept(List<Event> events) {
for (Event event : events) {
intercept(event);
}
return events;
}
@Override
public void close() {
// no-op
}
/**
* Builder which builds new instances of the TimestampInterceptor.
*/
public static class Builder implements Interceptor.Builder {
private boolean preserveExisting = DEFAULT_PRESERVE;
private String header = DEFAULT_HEADER_NAME;
@Override
public Interceptor build() {
return new TimestampInterceptor(preserveExisting, header);
}
@Override
public void configure(Context context) {
preserveExisting = context.getBoolean(CONFIG_PRESERVE, DEFAULT_PRESERVE);
header = context.getString(CONFIG_HEADER_NAME, DEFAULT_HEADER_NAME);
}
}
public static class Constants {
public static final String CONFIG_PRESERVE = "preserveExisting";
public static final boolean DEFAULT_PRESERVE = false;
public static final String CONFIG_HEADER_NAME = "headerName";
public static final String DEFAULT_HEADER_NAME = "timestamp";
}
}
二 、选择器
Channels选择器默认都是必须的,可选的通过配置来减少必须的。
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.flume;
import java.util.List;
import org.apache.flume.conf.Configurable;
/**
* <p>
* Allows the selection of a subset of channels from the given set based on
* its implementation policy. Different implementations of this interface
* embody different policies that affect the choice of channels that a source
* will push the incoming events to.
* </p>
*/
public interface ChannelSelector extends NamedComponent, Configurable {
/**
* @param channels all channels the selector could select from.
* 通道选择器可以选择的所有通道。
*/
public void setChannels(List<Channel> channels);
/**
* Returns a list of required channels. A failure in writing the event to
* these channels must be communicated back to the source that received this
* event.
* 返回所需频道的列表。无法将事件写入这些通道的失败必须传达回接收此事件的源。
* @param event
* @return the list of required channels that this selector has selected for
* the given event.
* 此选择器为其选择的所需频道列表给定事件。
*/
public List<Channel> getRequiredChannels(Event event);
/**
* Returns a list of optional channels. A failure in writing the event to
* these channels must be ignored.
* 返回可选频道的列表。将事件写入*这些通道失败将被忽略
* @param event
* @return the list of optional channels that this selector has selected for
* the given event.
* 此选择器为给定事件选择的可选通道的列表
*/
public List<Channel> getOptionalChannels(Event event);
/**
* @return the list of all channels that this selector is configured to work
* with.
* 此选择器配置为可以使用的所有通道的列表
*/
public List<Channel> getAllChannels();
}