1.Source
自己造一些数据传输到Source中去
public class source extends AbstractSource implements Configurable, PollableSource {
//定义数据传入的前缀跟后缀
private String prefix;
private String suffix;
@Override
public Status process() throws EventDeliveryException {
Status status = null;
try{
for(int i = 1;i<100;i++){
SimpleEvent event = new SimpleEvent();
//设置数据
event.setBody((prefix+i+suffix).getBytes());
//发送body到ChannelProcess
getChannelProcessor().processEvent(event);
}
//返回状态的结果
status = Status.READY;
}catch (Exception e){
//显示报错信息
e.printStackTrace();
status = Status.BACKOFF;
}finally {
}
return status;
}
@Override
public long getBackOffSleepIncrement() {
return 0;
}
@Override
public long getMaxBackOffSleepInterval() {
return 0;
}
@Override
public void configure(Context context) {
//不定义前缀,默认使用“ruozedata”
this.prefix=context.getString("prefix","ruozedata");
this.suffix=context.getString("suffix");
}
}
自定义Source配置文件
# 定义Agent
a1.sources = r1
a1.sinks = k1
a1.channels = c1
# 定义Source
a1.sources.r1.type =com.ruozedata.bigdata.flume.source
a1.sources.r1.suffix = ---ruozedata001
# 定义Channel
a1.channels.c1.type = memory
# 定义Sink
a1.sinks.k1.type = logger
# 定义配置关系
a1.sinks.k1.channel = c1
a1.sources.r1.channels = c1
2.Sink
接受从Channel来的数据,并打印出来或者存储到HDFS中
public class sink extends AbstractSink implements Configurable {
private static final Logger logger= LoggerFactory.getLogger(sink.class);
private String prefix;
private String suffix;
@Override
public Status process() throws EventDeliveryException {
Status status = null;
//获取channel数据
Channel channel = getChannel();
//获取事务
Transaction transaction = channel.getTransaction();
//开启事务
transaction.begin();
try{
Event event=null;
//需要判断event是否为空,不然测试时候报错
while(true){ //一直等待
event = channel.take();
if(event != null){ //event拿到数据时候,继续往下执行
break;
}
}
String body=new String(event.getBody());
logger.info(prefix+body+suffix);
transaction.commit();
status = Status.READY;
}catch (Exception e){
//回滚事务
transaction.rollback();
status = Status.BACKOFF;
}finally {
}
return status;
}
@Override
public void configure(Context context) {
this.prefix=context.getString("prefix","ruoedata");
this.suffix=context.getString("suffix");
}
}
自定义Sink配置文件
# 指定Agent
a1.sources = r1
a1.sinks = k1
a1.channels = c1
# 指定source
a1.sources.r1.type = netcat
a1.sources.r1.bind = localhost
a1.sources.r1.port = 5678
# 指定sink
a1.sinks.k1.type = com.ruozedata.bigdata.flume.sink
a1.sinks.k1.suffix = -JUMP
# 指定channel
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100
# 定义配置关系
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1
识别
下方二
维码
,即可关注公众号获取最新大厂技术
![](https://i-blog.csdnimg.cn/blog_migrate/3da85885cf34b3ca607410383aace768.png)