讲解一下如何自定义一个Flume的Source,很简单,下面是一个自定义Source,从Kafka中读取数据的Demo.
package death.flume;
import com.google.common.base.Preconditions;
import org.apache.flume.Context;
import org.apache.flume.EventDeliveryException;
import org.apache.flume.PollableSource;
import org.apache.flume.conf.Configurable;
import org.apache.flume.event.EventBuilder;
import org.apache.flume.source.AbstractSource;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import java.nio.charset.Charset;
import java.util.Collections;
import java.util.Properties;
// 继承AbstractSource抽象类,并实现Configurable接口
public class FlumeSourceDemo extends AbstractSource implements Configurable, PollableSource {
private String KAFKA_SERVER;
private String KAFKA_TOPIC;
private String GROUP_ID;
private String KEY_DES;
private String VALUE_DES;
private Properties props;
// Source的数据处理逻辑
@Override
public Status process() throws EventDeliveryException {
try {
KafkaConsumer<String, String> consumer = new KafkaConsumer<String, String>(props);
consumer.subscribe(Collections.singleton(KAFKA_TOPIC));
ConsumerRecords<String, String> records = consumer.poll(1000);
for (ConsumerRecord<String, String> record : records) {
System.out.println(record.value());
this.getChannelProcessor().processEvent(EventBuilder.withBody(record.value(), Charset.forName("UTF-8")));
}
return Status.READY;
}catch (Exception e){
e.printStackTrace();
return Status.BACKOFF;
}
}
//该方法用于Source的初始化
@Override
public void configure(Context context) {
KAFKA_SERVER = context.getString("kafka_server");
Preconditions.checkNotNull(KAFKA_SERVER, "kafka_server must be set");
KAFKA_TOPIC = context.getString("kafka_topic");
Preconditions.checkNotNull(KAFKA_TOPIC, "kafka_topic must be set");
GROUP_ID = context.getString("group_id");
Preconditions.checkNotNull(GROUP_ID, "group_id must be set");
KEY_DES = context.getString("key_des");
Preconditions.checkNotNull(KEY_DES, "key_des must be set");
VALUE_DES = context.getString("value_des");
Preconditions.checkNotNull(VALUE_DES, "value_des must be set");
}
// 该方法在Source启动时调用
@Override
public synchronized void start() {
super.start();
props = new Properties();
props.setProperty("bootstrap.servers", KAFKA_SERVER);
props.setProperty("group.id", GROUP_ID);
props.setProperty("key.deserializer", KEY_DES);
props.setProperty("value.deserializer", VALUE_DES);
}
// 该方法在Source停止时调用
@Override
public synchronized void stop() {
super.stop();
}
}