flume-自定义Sink基本框架

1，flume日志输出到控制台

bin/flume-ng agent --conf conf/ --name a1 --conf-file conf/xxx.properties -Dflume.root.logger=INFO,console -Dflume.monitoring.type=http -Dflume.monitoring.port=1234

2，flume日志输出到日志文件
bin/flume-ng agent --conf conf/ --name a1 --conf-file conf/xxx.properties -Dflume.root.logger=INFO,LOGFILE -Dflume.monitoring.type=http -Dflume.monitoring.port=1234

3，conf/flume-env.sh文件的配置JVM虚拟机内存

export JAVA_OPTS="-Xms1024m -Xmx2560m -Dcom.sun.management.jmxremote"

export JAVA_HOME=/opt/jdk1.7.0_79

4，flume执行文件

a1.sources = s1
a1.channels = c1
a1.sinks = k1

# define source
a1.sources.s1.type = exec
a1.sources.s1.command = tail -F /data/xxx/xxx.log
a1.sources.s1.shell = /bin/sh -c

#define channel
a1.channels.c1.type = memory
a1.channels.c1.capacity = 10000
a1.channels.c1.transactionCapacity = 4000

#a1.channels.c1.type = file
#a1.channels.c1.checkpointDir = /data/data1/flumedata/check
#a1.channels.c1.dataDirs = /data/data1/flumedata/datadir

#define selfhbase sinks
a1.sinks.k1.type=xxx.xxx.xxx.xxx
a1.sinks.k1.zoolist=cdh1:2181,cdh2:2181,cdh3:2181
a1.sinks.k1.tablename=xxxxx
a1.sinks.k1.bulkputsize=50

# define zuhe
a1.sources.s1.channels = c1
a1.sinks.k1.channel = c1

5，flume自定义sink问题

begin() called when transaction is OPEN!

//Flume-XXXsink的整体框架
public Status process() throws EventDeliveryException {
	Channel channel = getChannel();		
	try {	
		transaction = channel.getTransaction();
		transaction.begin();
					
		event = channel.take();
		if(event == null){
			break;
		}
		body = Bytes.toString(event.getBody());
		splits = body.split(",");
====================your code===========================
		transaction.commit();
	      return Status.READY;
	 } catch(IOException e){
		logger.error("ERROR Message "+e.toString());
		transaction.rollback(); // step 1
		return Status.BACKOFF; // step 2
	} catch (NullPointerException e){
		logger.info("ERROR Message null ==>"+e.toString());
		transaction.rollback(); // step 1
		return Status.BACKOFF; // step 2
	} catch (Throwable th){
 		transaction.rollback();
		if (th instanceof Error){
			throw (Error) th;
		}else{
		       throw new EventDeliveryException(th);
		}
	}finally {
		try {
			this.table.put(this.hbasePutList); 
		} catch (IOException e) {
			logger.error("", e);
		}
		this.hbasePutList.clear(); 
		transaction.close(); 	//一定是等流程处理完成后才close()

        }

注意，要在transaction.close()之前操作transaction.rollback()或是transaction.commit()