Flume-NG Configuration
上一节,看完了flume启动脚本,其中里面解析了各种用户传入的参数,并将其作为参数传给了 org.apache.flume.node.Application。 本节从 Application 入口类了解一下 flume-ng 如何启动。以及 flume-ng-configuration 模块是如何解析配置文件的。
org.apache.flume.node.Application
package org.apache.flume.node;
import com.google.common.base.Throwables;
import com.google.common.collect.Lists;
import com.google.common.eventbus.EventBus;
import com.google.common.eventbus.Subscribe;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.flume.Channel;
import org.apache.flume.Constants;
import org.apache.flume.Context;
import org.apache.flume.SinkRunner;
import org.apache.flume.SourceRunner;
import org.apache.flume.instrumentation.MonitorService;
import org.apache.flume.instrumentation.MonitoringType;
import org.apache.flume.lifecycle.LifecycleAware;
import org.apache.flume.lifecycle.LifecycleState;
import org.apache.flume.lifecycle.LifecycleSupervisor;
import org.apache.flume.lifecycle.LifecycleSupervisor.SupervisorPolicy;
import org.apache.flume.util.SSLUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import java.util.Map.Entry;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.locks.ReentrantLock;
public class Application {
private static final Logger logger = LoggerFactory
.getLogger(Application.class);
public static final String CONF_MONITOR_CLASS = "flume.monitoring.type";
public static final String CONF_MONITOR_PREFIX = "flume.monitoring.";
private final List<LifecycleAware> components;
private final LifecycleSupervisor supervisor;
private MaterializedConfiguration materializedConfiguration;
private MonitorService monitorServer;
private final ReentrantLock lifecycleLock = new ReentrantLock();
public Application() {
this(new ArrayList<LifecycleAware>(0));
}
public Application(List<LifecycleAware> components) {
this.components = components;
supervisor = new LifecycleSupervisor();
}
public void start() {
lifecycleLock.lock();
try {
for (LifecycleAware component : components) {
supervisor.supervise(component,
new SupervisorPolicy.AlwaysRestartPolicy(), LifecycleState.START);
}
} finally {
lifecycleLock.unlock();
}
}
@Subscribe
public void handleConfigurationEvent(MaterializedConfiguration conf) {
try {
lifecycleLock.lockInterruptibly();
stopAllComponents();
startAllComponents(conf);
} catch (InterruptedException e) {
logger.info("Interrupted while trying to handle configuration event");
return;
} finally {
// If interrupted while trying to lock, we don't own the lock, so must not attempt to unlock
if (lifecycleLock.isHeldByCurrentThread()) {
lifecycleLock.unlock();
}
}
}
public void stop() {
lifecycleLock.lock();
stopAllComponents();
try {
supervisor.stop();
if (monitorServer != null) {
monitorServer.stop();
}
} finally {
lifecycleLock.unlock();
}
}
private void stopAllComponents() {
if (this.materializedConfiguration != null) {
logger.info("Shutting down configuration: {}", this.materializedConfiguration);
for (Entry<String, SourceRunner> entry :
this.materializedConfiguration.getSourceRunners().entrySet()) {
try {
logger.info("Stopping Source " + entry.getKey());
supervisor.unsupervise(entry.getValue());
} catch (Exception e) {
logger.error("Error while stopping {}", entry.getValue(), e);
}
}
for (Entry<String, SinkRunner> entry :
this.materializedConfiguration.getSinkRunners().entrySet()) {
try {
logger.info("Stopping Sink " + entry.getKey());
supervisor.unsupervise(entry.getValue());
} catch (Exception e) {
logger.error("Error while stopping {}", entry.getValue(), e);
}
}
for (Entry<String, Channel> entry :
this.materializedConfiguration.getChannels().entrySet()) {
try {
logger.info("Stopping Channel " + entry.getKey());
supervisor.unsupervise(entry.getValue());
} catch (Exception e) {
logger.error("Error while stopping {}", entry.getValue(), e);
}
}
}
if (monitorServer != null) {
monitorServer.stop();
}
}
private void startAllComponents(MaterializedConfiguration materializedConfiguration) {
logger.info("Starting new configuration:{}", materializedConfiguration);
this.materializedConfiguration = materializedConfiguration;
for (Entry<String, Channel> entry :
materializedConfiguration.getChannels().entrySet()) {
try {
logger.info("Starting Channel " + entry.getKey());
supervisor.supervise(entry.getValue(),
new SupervisorPolicy.AlwaysRestartPolicy(), LifecycleState.START);
} catch (Exception e) {
logger.error("Error while starting {}", entry.getValue(), e);
}
}
/*
* Wait for all channels to start.
*/
for (Channel ch : materializedConfiguration.getChannels().values()) {
while (ch.getLifecycleState() != LifecycleState.START
&& !supervisor.isComponentInErrorState(ch)) {
try {
logger.info("Waiting for channel: " + ch.getName() +
" to start. Sleeping for 500 ms");
Thread.sleep(500);
} catch (InterruptedException e) {
logger.error("Interrupted while waiting for channel to start.", e);
Throwables.propagate(e);
}
}
}
for (Entry<String, SinkRunner> entry : materializedConfiguration.getSinkRunners().entrySet()) {
try {
logger.info("Starting Sink " + entry.getKey());
supervisor.supervise(entry.getValue(),
new SupervisorPolicy.AlwaysRestartPolicy(), LifecycleState.START);
} catch (Exception e) {
logger.error("Error while starting {}", entry.getValue(), e);
}
}
for (Entry<String, SourceRunner> entry :
materializedConfiguration.getSourceRunners().entrySet()) {
try {
logger.info("Starting Source " + entry.getKey());
supervisor.supervise(entry.getValue(),
new SupervisorPolicy.AlwaysRestartPolicy(), LifecycleState.START);
} catch (Exception e) {
logger.error("Error while starting {}", entry.getValue(), e);
}
}
this.loadMonitoring();
}
@SuppressWarnings("unchecked")
private void loadMonitoring() {
Properties systemProps = System.getProperties();
Set<String> keys = systemProps.stringPropertyNames();
try {
if (keys.contains(CONF_MONITOR_CLASS)) {
String monitorType = systemProps.getProperty(CONF_MONITOR_CLASS);
Class<? extends MonitorService> klass;
try {
//Is it a known type?
klass = MonitoringType.valueOf(
monitorType.toUpperCase(Locale.ENGLISH)).getMonitorClass();
} catch (Exception e) {
//Not a known type, use FQCN
klass = (Class<? extends MonitorService>) Class.forName(monitorType);
}
this.monitorServer = klass.newInstance();
Context context = new Context();
for (String key : keys) {
if (key.startsWith(CONF_MONITOR_PREFIX)) {
context.put(key.substring(CONF_MONITOR_PREFIX.length()),
systemProps.getProperty(key));
}
}
monitorServer.configure(context);
monitorServer.start();
}
} catch (Exception e) {
logger.warn("Error starting monitoring. "
+ "Monitoring might not be available.", e);
}
}
public static void main(String[] args) {
try {
SSLUtil.initGlobalSSLParameters();
Options options = new Options();
Option option = new Option("n", "name", true, "the name of this agent");
option.setRequired(true);
options.addOption(option);
option = new Option("f", "conf-file", true,
"specify a config file (required if -z missing)");
option.setRequired(false);
options.addOption(option);
option = new Option(null, "no-reload-conf", false,
"do not reload config file if changed");
options.addOption(option);
// Options for Zookeeper
option = new Option("z", "zkConnString", true,
"specify the ZooKeeper connection to use (required if -f missing)");
option.setRequired(false);
options.addOption(option);
option = new Option("p", "zkBasePath", true,
"specify the base path in ZooKeeper for agent configs");
option.setRequired(false);
options.addOption(option);
option = new Option("h", "help", false, "display help text");
options.addOption(option);
CommandLineParser parser = new GnuParser();
CommandLine commandLine = parser.parse(options, args);
if (commandLine.hasOption('h')) {
new HelpFormatter().printHelp("flume-ng agent", options, true);
return;
}
String agentName = commandLine.getOptionValue('n');
boolean reload = !commandLine.hasOption("no-reload-conf");
boolean isZkConfigured = false;
if (commandLine.hasOption('z') || commandLine.hasOption("zkConnString")) {
isZkConfigured = true;
}
Application application;
if (isZkConfigured) {
// get options
String zkConnectionStr = commandLine.getOptionValue('z');
String baseZkPath = commandLine.getOptionValue('p');
if (reload) {
EventBus eventBus = new EventBus(agentName + "-event-bus");
List<LifecycleAware> components = Lists.newArrayList();
PollingZooKeeperConfigurationProvider zookeeperConfigurationProvider =
new PollingZooKeeperConfigurationProvider(
agentName, zkConnectionStr, baseZkPath, eventBus);
components.add(zookeeperConfigurationProvider);
application = new Application(components);
eventBus.register(application);
} else {
StaticZooKeeperConfigurationProvider zookeeperConfigurationProvider =
new StaticZooKeeperConfigurationProvider(
agentName, zkConnectionStr, baseZkPath);
application = new Application();
application.handleConfigurationEvent(zookeeperConfigurationProvider.getConfiguration());
}
} else {
File configurationFile = new File(commandLine.getOptionValue('f'));
/*
* The following is to ensure that by default the agent will fail on
* startup if the file does not exist.
*/
if (!configurationFile.exists()) {
// If command line invocation, then need to fail fast
if (System.getProperty(Constants.SYSPROP_CALLED_FROM_SERVICE) ==
null) {
String path = configurationFile.getPath();
try {
path = configurationFile.getCanonicalPath();
} catch (IOException ex) {
logger.error("Failed to read canonical path for file: " + path,
ex);
}
throw new ParseException(
"The specified configuration file does not exist: " + path);
}
}
List<LifecycleAware> components = Lists.newArrayList();
if (reload) {
EventBus eventBus = new EventBus(agentName + "-event-bus");
PollingPropertiesFileConfigurationProvider configurationProvider =
new PollingPropertiesFileConfigurationProvider(
agentName, configurationFile, eventBus, 30);
components.add(configurationProvider);
application = new Application(components);
eventBus.register(application);
} else {
PropertiesFileConfigurationProvider configurationProvider =
new PropertiesFileConfigurationProvider(agentName, configurationFile);
application = new Application();
application.handleConfigurationEvent(configurationProvider.getConfiguration());
}
}
application.start();
final Application appReference = application;
Runtime.getRuntime().addShutdownHook(new Thread("agent-shutdown-hook") {
@Override
public void run() {
appReference.stop();
}
});
} catch (Exception e) {
logger.error("A fatal error occurred while running. Exception follows.", e);
}
}
}
flume 一共有两种读取配置文件方式,一种是通过-z / --zkConnString 从zookeeper中读取配置文件,一种是通过-f / --conf-file 从本地文件系统读取配置文件,配置文件格式遵循了 java properties 文件的格式。我们可以将多个 agent 定义在同一个配置文件中, 配置文件中包含了所需要的 source, sink, channel 组件,并且还要说明他们是怎么连接到一起的。
flume 启动的时候有一个叫做 --no-reload-conf 的配置项,这个配置的作用是当flume配置文件变动时是否可以热加载修改后的配置文件,而不用重启flume。
这里我们从本地文件系统读取配置文件,主要代码如下:
// 获取配置文件对象
File configurationFile = new File(commandLine.getOptionValue('f'));
/*
* 首先检查一下配置文件是否存在,不存在的话直接抛异常
*/
if (!configurationFile.exists()) {
// If command line invocation, then need to fail fast
if (System.getProperty(Constants.SYSPROP_CALLED_FROM_SERVICE) ==
null) {
String path = configurationFile.getPath();
try {
path = configurationFile.getCanonicalPath();
} catch (IOException ex) {
logger.error("Failed to read canonical path for file: " + path,
ex);
}
throw new ParseException(
"The specified configuration file does not exist: " + path);
}
}
List<LifecycleAware> components = Lists.newArrayList();
if (reload) {
//谷歌 guava 事件总线有兴趣的可以了解下(我还没看) ,大概的意思就是 30 秒轮询一次配置文件
EventBus eventBus = new EventBus(agentName + "-event-bus");
PollingPropertiesFileConfigurationProvider configurationProvider =
new PollingPropertiesFileConfigurationProvider(
agentName, configurationFile, eventBus, 30);
components.add(configurationProvider);
application = new Application(components);
eventBus.register(application);
} else {
PropertiesFileConfigurationProvider configurationProvider =
new PropertiesFileConfigurationProvider(agentName, configurationFile);
application = new Application();
application.handleConfigurationEvent(configurationProvider.getConfiguration());
}
可以看到这里面解析配置文件由 PollingPropertiesFileConfigurationProvider 或者 PropertiesFileConfigurationProvider 两个类来完成,具体要看是否启用了 no-reload-conf 配置。但是最终都是要去实例化 FlumeConfiguration 这个类。
public FlumeConfiguration(Map<String, String> properties) {
agentConfigMap = new HashMap<>();
errors = new LinkedList<>();
// Construct the in-memory component hierarchy
for (Entry<String, String> entry : properties.entrySet()) {
if (!addRawProperty(entry.getKey(), entry.getValue())) {
LOGGER.warn("Configuration property ignored: {} = {}", entry.getKey(), entry.getValue());
}
}
// Now iterate thru the agentContext and create agent configs and add them
// to agentConfigMap
// validate and remove improperly configured components
validateConfiguration();
}
FlumeConfiguration 构造函数如上,主要是将配置文件中的 key , value 遍历出来,然后调用 addRawProperty 方法将他们添加进FlumeConfiguration的成员变量agentConfigMap中。
private boolean addRawProperty(String rawName, String rawValue) {
// Null names and values not supported
if (rawName == null || rawValue == null) {
addError("", AGENT_NAME_MISSING, ERROR);
return false;
}
// Remove leading and trailing spaces
String name = rawName.trim();
String value = rawValue.trim();
// Empty values are not supported
if (value.isEmpty()) {
addError(name, PROPERTY_VALUE_NULL, ERROR);
return false;
}
//获取第一个"."(点)的位置
int index = name.indexOf('.');
// All configuration keys must have a prefix defined as agent name
if (index == -1) {
addError(name, AGENT_NAME_MISSING, ERROR);
return false;
}
//获取 agent 名字
String agentName = name.substring(0, index);
// Agent name must be specified for all properties
if (agentName.isEmpty()) {
addError(name, AGENT_NAME_MISSING, ERROR);
return false;
}
//获取 . 后面的字符串
String configKey = name.substring(index + 1);
// Configuration key must be specified for every property
if (configKey.isEmpty()) {
addError(name, PROPERTY_NAME_NULL, ERROR);
return false;
}
//当agentConfigMap中不存在当前agent, 将解析出来的 agent 名字 添加到agentConfigMap中,同时也将flumeConfiguration中的AgentConfiguration对象也实例化出来
AgentConfiguration aconf = agentConfigMap.get(agentName);
if (aconf == null) {
aconf = new AgentConfiguration(agentName, errors);
agentConfigMap.put(agentName, aconf);
}
//最后还需要继续解析 agent 名字后面的配置项
return aconf.addProperty(configKey, value);
}
private boolean addProperty(String key, String value) {
//这里分别要判断配置项是属于哪一类: configfilter , sources, sinks, channels, sinkgroups
// Check for configFilters
//判断是不是 <agentName>.configfilters = xxx 形式
if (CONFIG_CONFIGFILTERS.equals(key)) {
if (configFilters == null) {
configFilters = value;
return true;
} else {
LOGGER.warn("Duplicate configfilter list specified for agent: {}", agentName);
addError(CONFIG_CONFIGFILTERS, DUPLICATE_PROPERTY, ERROR);
return false;
}
}
// Check for sources
//判断是不是 <agentName>.sources = xxx 形式
if (CONFIG_SOURCES.equals(key)) {
if (sources == null) {
sources = value;
return true;
} else {
LOGGER.warn("Duplicate source list specified for agent: {}", agentName);
addError(CONFIG_SOURCES, DUPLICATE_PROPERTY, ERROR);
return false;
}
}
// Check for sinks
//判断是不是 <agentName>.sinks = xxx 形式
if (CONFIG_SINKS.equals(key)) {
if (sinks == null) {
sinks = value;
LOGGER.info("Added sinks: {} Agent: {}", sinks, agentName);
return true;
} else {
LOGGER.warn("Duplicate sink list specfied for agent: {}", agentName);
addError(CONFIG_SINKS, DUPLICATE_PROPERTY, ERROR);
return false;
}
}
// Check for channels
//判断是不是 <agentName>.channels = xxx 形式
if (CONFIG_CHANNELS.equals(key)) {
if (channels == null) {
channels = value;
return true;
} else {
LOGGER.warn("Duplicate channel list specified for agent: {}", agentName);
addError(CONFIG_CHANNELS, DUPLICATE_PROPERTY, ERROR);
return false;
}
}
// Check for sinkgroups
//判断是不是 <agentName>.sinkgroups = xxx 形式
if (CONFIG_SINKGROUPS.equals(key)) {
if (sinkgroups == null) {
sinkgroups = value;
return true;
} else {
LOGGER.warn("Duplicate sinkgroup list specfied for agent: {}", agentName);
addError(CONFIG_SINKGROUPS, DUPLICATE_PROPERTY, ERROR);
return false;
}
}
// 以上五种形式会直接将 <agentName>. 后面的字符串传到 AgentConfiguration 数据结构中对应字段中
// 以上五个组件 configfilter、sources、channels、sinks、sinkgroups 可以称为 component.
//然后会调用 addAsXXXConfig 方法来检查这一行配置文件是以什么开头的,其实最后调用的都是
//addComponentConfig方法。这个方法会将后面的参数传入到AgentConfiguration 的各种 context 的 Map中。
if (addAsSourceConfig(key, value)
|| addAsChannelValue(key, value)
|| addAsSinkConfig(key, value)
|| addAsSinkGroupConfig(key, value)
|| addAsConfigFilterConfig(key, value)
) {
return true;
}
LOGGER.warn("Invalid property specified: {}", key);
addError(key, INVALID_PROPERTY, ERROR);
return false;
}
private boolean addComponentConfig(
String key, String value, String configPrefix, Map<String, Context> contextMap
) {
ComponentNameAndConfigKey parsed = parseConfigKey(key, configPrefix);
if (parsed != null) {
String name = parsed.getComponentName().trim();
LOGGER.info("Processing:{}", name);
Context context = contextMap.get(name);
if (context == null) {
LOGGER.debug("Created context for {}: {}", name, parsed.getConfigKey());
context = new Context();
contextMap.put(name, context);
}
context.put(parsed.getConfigKey(), value);
return true;
}
return false;
}
看到这里感觉 flume 配置文件就是按树状结构一层一层的去解析。这里有个小问题就是使用java Properties类读取 properties 文件,不是按顺序从头到尾。所以这里面就依靠各种名字去匹配对应关系,FlumeConfiguration的数据结构中体现为 Map<String, AgentConfiguration> agentConfigMap
,以及AgentConfiguration中的一系列的名字后面会详细的看一下。