Flume自定义sink
1.介绍
Sink不断地轮询Channel 中的事件且批量地移除它们,并将这些事件批量写入到存储或索引系统、或者被发送到另一个Flume Agent。
Sink是完全事务性的。在从Channel 批量删除数据之前,每个Sink用Channel启动一个事务。批量事件一旦成功写出到存储系统或下一个Flume Agent,Sink就利用Channel提交事务。事务一旦被提交,该Channel从自己的内部缓冲区删除事件。
Sink组件目的地包括hdfs、logger、avro、thrift、ipc、file、null、HBase、solr、自定义。官方提供的Sink类型已经很多,但是有时候并不能满足实际开发当中的需求,此时我们就需要根据实际需求自定义某些Sink。
编写的大概流程
代码
import org.apache.flume.*;
import org.apache.flume.conf.Configurable;
import org.apache.flume.sink.AbstractSink;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class MySink extends AbstractSink implements Configurable {
//定义俩个属性 前后缀
private String prefix;
private String subfix;
/**
* 获取channel
* 从channel获取事务及数据
* 发送数据
*
* @return
* @throws EventDeliveryException
*/
@Override
public Status process() throws EventDeliveryException {
//获取log对象
Logger logger = LoggerFactory.getLogger(MySink.class);
//定义返回值
Status status = null;
//获取channel
Channel channel = getChannel();
//从channel获取事务
Transaction transaction = channel.getTransaction();
//开启事务
transaction.begin();
try {
//从channel获取数据
Event event = channel.take();
//处理事务
if (event != null) {
String body = new String(event.getBody());
logger.info(prefix+body+subfix);
}
//提交事务
transaction.commit();
//成功提交修改状态信息
status = Status.READY;
} catch (ChannelException e) {
e.printStackTrace();
//提交失败,回滚,状态修改
transaction.rollback();
status = Status.BACKOFF;
} finally {
//关闭
transaction.close();
}
//返回状态信息
return status;
}
@Override
public void configure(Context context) {
//读取配置文件,给前后缀赋值
prefix = context.getString("prefix");
subfix = context.getString("prefix", "subfix");
}
}
配置文件
a2.sources = r1
a2.sinks = k1
a2.channels = c1
# Describe/configure the source
a2.sources.r1.type = avro
a2.sources.r1.bind = hadoop102
a2.sources.r1.port = 4141
# Describe the sink
a2.sinks.k1.type = flume.MySink
a2.sinks.k1.prefix = aa
a2.sinks.k1.subfix = bb
# Describe the channel
a2.channels.c1.type = memory
a2.channels.c1.capacity = 1000
a2.channels.c1.transactionCapacity = 100
# Bind the source and sink to the channel
a2.sources.r1.channels = c1
a2.sinks.k1.channel = c1