自定义flume-source

最新推荐文章于 2024-08-06 22:24:16 发布

卡奥斯道

最新推荐文章于 2024-08-06 22:24:16 发布

阅读量835

点赞数

分类专栏： flume 文章标签： flume source 自定义

本文链接：https://blog.csdn.net/kaaosidao/article/details/78178138

版权

flume 专栏收录该内容

4 篇文章 0 订阅

订阅专栏

package com.bigdata.flume;

import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.EventDeliveryException;
import org.apache.flume.PollableSource;
import org.apache.flume.conf.Configurable;
import org.apache.flume.event.EventBuilder;
import org.apache.flume.source.AbstractSource;

import java.nio.charset.Charset;
import java.util.HashMap;
import java.util.Random;

/**
 * 
 */
public class MySource extends AbstractSource implements Configurable, PollableSource {

    @Override
    public void configure(Context context) {

    }

    @Override
    public Status process() throws EventDeliveryException {
        Status status = null;

        try{
            Random random = new Random();
            int randomNum = random.nextInt(100);
            String text = "Hello World :" + random.nextInt(100);
            HashMap<String, String> header = new HashMap<String, String>();
            header.put("id", random+"");
            this.getChannelProcessor()
                    .processEvent(EventBuilder.withBody(text, Charset.forName("UTF-8"), header));

            status = Status.READY;
        }catch (Throwable t) {
            // Log exception, handle individual exceptions as needed

            status = Status.BACKOFF;

            // re-throw all Errors
            if (t instanceof Error) {
                throw (Error)t;
            }
        }
        return status;
    }

}

2.flume-source.conf

#
a1.sources=r1
a1.sinks=k1
a1.channels=c1
  
#
a1.sources.r1.type=com.bigdata.flume.MySource
  
#
a1.channels.c1.type=memory
a1.channels.c1.capacity=1000
a1.channels.c1.transactionCapacity=100
a1.channels.c1.byteCapacityBufferPercentage=20
a1.channels.c1.byteCapacity=800000
  
#
a1.sinks.k1.type=file_roll
#注意目录得是虚拟机上的
a1.sinks.k1.sink.directory=/tmp/a
  
# 
a1.sources.r1.channels=c1
a1.sinks.k1.channel=c1

#运行命令bin/flume-ng agent --conf conf --name a1 --conf-file conf/fulme-source.conf -Dflume.root.logger=INFO,console

3.结果