Flume拦截器与选择器

一 、拦截器

1. 拦截器接口

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.flume.interceptor;

import java.util.List;
import org.apache.flume.Event;
import org.apache.flume.annotations.InterfaceAudience;
import org.apache.flume.annotations.InterfaceStability;
import org.apache.flume.conf.Configurable;

@InterfaceAudience.Public
@InterfaceStability.Stable
public interface Interceptor {
  /**
   * Any initialization / startup needed by the Interceptor.
   * 在启动拦截器时做的一些初始化操作
   */
  public void initialize();

  /**
   * Interception of a single {@link Event}.
   * @param event Event to be intercepted
   * @return Original or modified event, or {@code null} if the Event
   * is to be dropped (i.e. filtered out).
   * 每次取一个事件进行事务操作
   */
  public Event intercept(Event event);

  /**
   * Interception of a batch of {@linkplain Event events}.
   * @param events Input list of events
   * @return Output list of events. The size of output list MUST NOT BE GREATER
   * than the size of the input list (i.e. transformation and removal ONLY).
   * Also, this method MUST NOT return {@code null}. If all events are dropped,
   * then an empty List is returned.
   * 每次取一个集合的事务操作
   */
  public List<Event> intercept(List<Event> events);

  /**
   * Perform any closing / shutdown needed by the Interceptor.
   * 主要是对在初始化时的一些需要关闭的操作
   */
  public void close();

  /** Builder implementations MUST have a no-arg constructor
	* 对拦截器内部功能需要用到的一些参数的配置
	*/
  public interface Builder extends Configurable {
    public Interceptor build();
  }
}

通过上面的接口文档可以知道拦截器的实现很简单，实现对应的方法就可以了。注意拦截器的处理逻辑必须确保是线程安全的。

2. 时间戳拦截器源码分析

   时间戳拦截器是官方提供的源码通过它来加深对拦截器的理解消化。

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.flume.interceptor;

import java.util.List;
import java.util.Map;
import org.apache.flume.Context;
import org.apache.flume.Event;

import static org.apache.flume.interceptor.TimestampInterceptor.Constants.*;

/**
 * Simple Interceptor class that sets the current system timestamp on all events
 * that are intercepted.
 * By convention, this timestamp header is named "timestamp" by default and its format
 * is a "stringified" long timestamp in milliseconds since the UNIX epoch.
 * The name of the header can be changed through the configuration using the
 * config key "header".
 */
public class TimestampInterceptor implements Interceptor {

  private final boolean preserveExisting;
  private final String header;

  /**
   * Only {@link TimestampInterceptor.Builder} can build me
   */
  private TimestampInterceptor(boolean preserveExisting, String header) {
    this.preserveExisting = preserveExisting;
    this.header = header;
  }

  @Override
  public void initialize() {
    // no-op
  }

  /**
   * Modifies events in-place.
   */
  @Override
  public Event intercept(Event event) {
    Map<String, String> headers = event.getHeaders();
    if (preserveExisting && headers.containsKey(header)) {
      // we must preserve the existing timestamp
    } else {
      long now = System.currentTimeMillis();
      headers.put(header, Long.toString(now));
    }
    return event;
  }

  /**
   * Delegates to {@link #intercept(Event)} in a loop.
   * @param events
   * @return
   */
  @Override
  public List<Event> intercept(List<Event> events) {
    for (Event event : events) {
      intercept(event);
    }
    return events;
  }

  @Override
  public void close() {
    // no-op
  }

  /**
   * Builder which builds new instances of the TimestampInterceptor.
   */
  public static class Builder implements Interceptor.Builder {

    private boolean preserveExisting = DEFAULT_PRESERVE;
    private String header = DEFAULT_HEADER_NAME;

    @Override
    public Interceptor build() {
      return new TimestampInterceptor(preserveExisting, header);
    }

    @Override
    public void configure(Context context) {
      preserveExisting = context.getBoolean(CONFIG_PRESERVE, DEFAULT_PRESERVE);
      header = context.getString(CONFIG_HEADER_NAME, DEFAULT_HEADER_NAME);
    }

  }

  public static class Constants {
    public static final String CONFIG_PRESERVE = "preserveExisting";
    public static final boolean DEFAULT_PRESERVE = false;
    public static final String CONFIG_HEADER_NAME = "headerName";
    public static final String DEFAULT_HEADER_NAME = "timestamp";
  }

}

二 、选择器

Channels选择器默认都是必须的，可选的通过配置来减少必须的。

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.flume;

import java.util.List;

import org.apache.flume.conf.Configurable;

/**
 * <p>
 * Allows the selection of a subset of channels from the given set based on
 * its implementation policy. Different implementations of this interface
 * embody different policies that affect the choice of channels that a source
 * will push the incoming events to.
 * </p>
 */
public interface ChannelSelector extends NamedComponent, Configurable {

  /**
   * @param channels all channels the selector could select from.
   * 通道选择器可以选择的所有通道。
   */
  public void setChannels(List<Channel> channels);

  /**
   * Returns a list of required channels. A failure in writing the event to
   * these channels must be communicated back to the source that received this
   * event.
   * 返回所需频道的列表。无法将事件写入这些通道的失败必须传达回接收此事件的源。
   * @param event
   * @return the list of required channels that this selector has selected for
   * the given event.
   * 此选择器为其选择的所需频道列表给定事件。
   */
  public List<Channel> getRequiredChannels(Event event);


  /**
   * Returns a list of optional channels. A failure in writing the event to
   * these channels must be ignored.
   * 返回可选频道的列表。将事件写入*这些通道失败将被忽略
   * @param event
   * @return the list of optional channels that this selector has selected for
   * the given event.
   * 此选择器为给定事件选择的可选通道的列表
   */
  public List<Channel> getOptionalChannels(Event event);

  /**
   * @return the list of all channels that this selector is configured to work
   * with.
   * 此选择器配置为可以使用的所有通道的列表
   */
  public List<Channel> getAllChannels();

}