日志采集与处理 flume拦截器--使用Aviator扩展拦截器功能

1. 背景

在使用flume进行日志收集、处理的过程中，发现flume带的拦截器处理能力不够。笔者开发的大数据平台XSailboat中广泛使用了Aviator表达式，可以通过使用Aviator表达式提升拦截器的处理能力。

2. 代码

pom.xml

...
	<dependency>
		<groupId>com.googlecode.aviator</groupId>
		<artifactId>aviator</artifactId>
		<version>5.3.3</version>
	</dependency>
...

类：org.apache.flume.interceptor.extend.AviatorInterceptor(没有用自己的包)

package org.apache.flume.interceptor.extend;

import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.flume.Event;
import org.apache.flume.interceptor.Interceptor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.google.common.base.Charsets;
import com.google.common.collect.Lists;
import com.googlecode.aviator.Expression;

public class AviatorInterceptor implements Interceptor
{

	static final Logger sLogger = LoggerFactory.getLogger(AviatorInterceptor.class);
	
	Expression mHandlerExpr ;
	Expression mFilterExpr ;

	AviatorInterceptor(Expression aHandlerExpr, Expression aFilterExpr)
	{
		mHandlerExpr = aHandlerExpr ;
		mFilterExpr = aFilterExpr ;
	}

	@Override
	public void initialize()
	{

	}

	@Override
	public void close()
	{
		// NO-OP...
	}

	@Override
	public Event intercept(Event aEvent)
	{
		Map<String, String> headers = aEvent.getHeaders() ;
		String body = new String(aEvent.getBody(), Charsets.UTF_8) ;
		Map<String , Object> ctxMap = new HashMap<String, Object>() ;
		ctxMap.put("headers" , headers) ;
		ctxMap.put("body" , body) ;
		// 使用过滤器进行过滤，true放行
		if(mFilterExpr == null || Boolean.TRUE.equals(toBoolean(mFilterExpr.execute(ctxMap) , false)))
		{
			if(mHandlerExpr != null)
			{
				// 使用处理器进行处理
				Object result = mHandlerExpr.execute(ctxMap) ;
				aEvent.setHeaders((Map<String, String>) ctxMap.get("headers"));
				// 表达式返回的结果不为null，就把它作为消息内容
				if(result != null)
				{
					aEvent.setBody(result.toString().getBytes(Charsets.UTF_8)) ;
				}
			}
			return aEvent ;
		}
		return null ;
	}

	@Override
	public List<Event> intercept(List<Event> events)
	{
		List<Event> intercepted = Lists.newArrayListWithCapacity(events.size());
		for (Event event : events)
		{
			Event interceptedEvent = intercept(event);
			if (interceptedEvent != null)
			{
				intercepted.add(interceptedEvent);
			}
		}
		return intercepted;
	}
	
	static Boolean toBoolean(Object aVal , boolean aQuietly)
	{
		if(aVal == null)
			return null ;
		if(aVal instanceof Boolean)
			return (Boolean)aVal ;
		if(aVal instanceof String)
		{
			if(((String)aVal).isEmpty())
				return null ;
			return "true".equalsIgnoreCase((String)aVal) ;
		}
		if(aVal instanceof Number)
			return ((Number)aVal).intValue() > 0 ;
		if(aQuietly)
			return null ;
		throw new IllegalArgumentException(String.format("不支持将%s类型的对象转化成Boolean" 
				, aVal.getClass().getName())) ;
	}
}

类：org.apache.flume.interceptor.extend.AviatorInterceptorBuilder

package org.apache.flume.interceptor.extend;

import org.apache.flume.Context;
import org.apache.flume.interceptor.Interceptor;

import com.googlecode.aviator.AviatorEvaluator;
import com.googlecode.aviator.Expression;

public class AviatorInterceptorBuilder implements Interceptor.Builder
{
	static final String HANDLER = "handler";
	static final String FILTER = "filter";

	Expression mHandlerExpr ;
	
	Expression mFilterExpr ;

	@Override
	public void configure(Context context)
	{
		String handlerStr = context.getString(HANDLER);
		if(handlerStr != null && handlerStr.length() >0)
		{
			mHandlerExpr = AviatorEvaluator.compile(handlerStr) ;
		}
		String filterStr = context.getString(FILTER);
		if(filterStr != null && !filterStr.isEmpty())
		{
			mFilterExpr = AviatorEvaluator.compile(filterStr) ;
		}
	}

	@Override
	public Interceptor build()
	{
		if(mHandlerExpr != null || mFilterExpr != null)
			return new AviatorInterceptor(mHandlerExpr , mFilterExpr) ;
		else
			throw new IllegalStateException("未配置handler或filter") ;
	}
}

3. 配置示例

...
_log_flink.sources.r1.interceptors = i1
_log_flink.sources.r1.interceptors.i1.type = org.apache.flume.interceptor.extend.AviatorInterceptorBuilder
_log_flink.sources.r1.interceptors.i1.handler = let i = string.indexOf(body , " ") ; \
if(i>0){ \
	let key = string.substring(body , 0 , i) + "." + seq.get(headers , "offset") ; \
	let msg = string.substring(body , i+1) ; \
	seq.put(headers , "hbase_rowkey" , key) ; \
	return msg ; \
} \
return nil ;
# _log_flink.sources.r1.interceptors.i1.filter = 表达式，过滤器，返回true或false
...