morphline java_介绍如何重写Flume-NG-morphline-avro-sink

修改源代码

1、  MorphlineHandler.java

添加setRpcClient接口：

public void setRpcClient(RpcClient client);

1、  MorphlineHandlerImpl.java

a)         定义线程池中线程的个数的私有成员变量

private static final int NUM =3;//线程池中线程个数定义为3，根据实际情况而定。定义线程池:

public ExcutorService es = Excutors.newFixedThreadPool(NUM);

b)         添加Override setRpcClient方法

@Override

public void setRpcClient(RpcClient client){

this.client = client;

}

c)         构建Collector内部类，其中定义一个队列，用来存储被处理完成的Record。

public static final class Collector implements Command {

public Collector(){}

private BlockingQueue queue = new ArrayBlockingQueue(10);

@Override

public Command getParent() {

return null;

}

public BlockingQueue getQueue() {

return queue;

}

public void setQueue(BlockingQueue queue) {

this.queue = queue;

}

@Override

public void notify(Record notification) {

}

@Override

public boolean process(Record record) {

Preconditions.checkNotNull(record);

try {

//接收到Record后，将其压如队列

queue.put(record);

} catch (InterruptedException e) {

e.printStackTrace();

return false;

}

return true;

}

}

d)         构建处理利用client发送event的内部线程

public class MorphlineTask implements Runnable {

private BlockingQueue queue;

private RpcClient client;

public MorphlineTask() {

}

public MorphlineTask(BlockingQueue queue, RpcClient client) {

this.queue = queue;

this.client = client;

}

@Override

public void run() {

Record r = null;

while (true) {

try {

//从队列中取出Record，并转换成event。

r = queue.take();

Map headers = null;

headers = new HashMap();

ListMultimap lmt = r.getFields();

Map> m = lmt.asMap();

Iterator it = m.entrySet().iterator();

while (it.hasNext()) {

Entry entry = (Entry) it

.next();

if (entry.getValue() != null && !entry.getKey().equals(Fields.ATTACHMENT_BODY)) {

List v = (List) entry.getValue();

if (v.get(0) != null) {

headers.put(entry.getKey(), v.get(0).toString());

}

}

}

try{

Event e = EventBuilder.withBody((byte[])r.getFirstValue(Fields.ATTACHMENT_BODY), headers);

client.append(e);

}catch(NullPointerException e){

e.printStackTrace();

LOG.error("Rpc Client is null!");

}

} catch (InterruptedException e) {

// TODO Auto-generated catch block

e.printStackTrace();

break;

} catch (EventDeliveryException e1) {

// TODO Auto-generated catch block

e1.printStackTrace();

}

}

}

}

e)         修改configure(Context context)方法

Collector finalChild = null;

try{

//初始化内部处理Record的Command

finalChild = new Collector();

}catch(Exception e){

e.printStackTrace();

}

//执行线程池中的线程，发送event

for (int i = 0; i < NUM; i++) {

es.execute(new MorphlineTask(finalChild.getQueue(), client));

}

3、  MorphlineSink.java

a)         将该类原来集成的AbstractSink修改成AbstractRpcSink；

b)         定义私有成员变量RpcClient client；

c)         定义私有成员变量Properties clientProps；

d)         修改configure(Context context)方法。

//创建clientProps。为client提供初始化数据

clientProps = new Properties();    clientProps.setProperty(RpcClientConfigurationConstants.CONFIG_HOSTS, "h1");   clientProps.setProperty(RpcClientConfigurationConstants.CONFIG_HOSTS_PREFIX +"h1", context.getString("hostname") + ":" + context.getInteger("port"));

for (Entry entry: context.getParameters().entrySet()) {      clientProps.setProperty(entry.getKey(),entry.getValue());

}

e)         修改start()方法

添加初始化client代码

//初始化RpcClient。

try {

client = initializeRpcClient(clientProps);

} catch (Exception e) {

e.printStackTrace();

}

赋值client

在构建handler部分添加赋值client代码

tmpHandler.setRpcClient(client);

f)         修改stop()方法

finally中添加client.close();

至此Flume-NG-Morphline-Avro-Sink编写完成。