flume-ng-sdk-1.8.0.jar 中依赖的jar 是 avro-1.7.4.jar,如果把avro 包换成最新的avro-1.8.2.jar 会导致Avro RPC client发送消息时失败,失败的异常如下:
Caused by: java.util.concurrent.ExecutionException: java.lang.AbstractMethodError: org.apache.avro.specific.SpecificFixed.getSchema()Lorg/apache/avro/Schema;
at java.util.concurrent.FutureTask.report(FutureTask.java:122)
at java.util.concurrent.FutureTask.get(FutureTask.java:206)
at org.apache.flume.api.NettyAvroRpcClient.append(NettyAvroRpcClient.java:280)
... 3 more
Caused by: java.lang.AbstractMethodError: org.apache.avro.specific.SpecificFixed.getSchema()Lorg/apache/avro/Schema;
at org.apache.avro.specific.SpecificFixed.<init>(SpecificFixed.java:36)
at org.apache.avro.ipc.MD5.<init>(MD5.java:16)
at org.apache.avro.ipc.Requestor.writeHandshake(Requestor.java:200)
at org.apache.avro.ipc.Requestor.access$300(Requestor.java:52)
at org.apache.avro.ipc.Requestor$Request.getBytes(Requestor.java:478)
at org.apache.avro.ipc.Requestor.request(Requestor.java:147)
at org.apache.avro.ipc.Requestor.request(Requestor.java:129)
at org.apache.avro.ipc.specific.SpecificRequestor.invoke(SpecificRequestor.java:84)
at com.sun.proxy.$Proxy2.append(Unknown Source)
at org.apache.flume.api.NettyAvroRpcClient$1.call(NettyAvroRpcClient.java:271)
at org.apache.flume.api.NettyAvroRpcClient$1.call(NettyAvroRpcClient.java:267)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
解决办法是:使用avro-1.7.4.jar 包才能导致不报错,但是如果之前用最新的avro.1.8.2.jar生成过avro schema对应的java类,那么那些类又该报错了,很无奈,只能根据avro-1.7.4.jar包重新生成了。
今天在学习Flume 的时候,照着官网给的例子,搭建了一个Source 是avro,Sink 端是logger的flume agent时报错了,客户端的avro消息无法通过RPC发送到flume agent,折腾了半天发现是avro 包导致的异常。
下面是我所使用的代码以及配置:
1. Flume agent 配置如下:
a1.channels = c1
a1.sources = r1
a1.sinks = k1
a1.channels.c1.type = memory
a1.sources.r1.channels = c1
a1.sources.r1.type = avro
a1.sources.r1.bind = 0.0.0.0
a1.sources.r1.port = 41414
a1.sinks.k1.channel = c1
a1.sinks.k1.type = logger
2. 启动Flume Agent:
./bin/flume-ng agent --conf conf/ --conf-file conf/flume-avro.conf --name a1 -Dflume.root.logger=INFO,console
启动成功:
3. 客户端代码如下:
package com.learn.flume; import org.apache.flume.Event; import org.apache.flume.EventDeliveryException; import org.apache.flume.api.NettyAvroRpcClient; import org.apache.flume.api.RpcClient; import org.apache.flume.api.RpcClientFactory; import org.apache.flume.event.EventBuilder; import java.nio.charset.Charset; import java.util.Properties; public class MyApp { public static void main(String[] args) { MyRpcClientFacade client = new MyRpcClientFacade(); // Initialize client with the remote Flume agent's host and port client.init("localhost", 41414); // Send 10 events to the remote Flume agent. That agent should be // configured to listen with an AvroSource. String sampleData = "Hello Flume!"; for (int i = 0; i < 10; i++) { client.sendDataToFlume(sampleData); } client.cleanUp(); } } class MyRpcClientFacade { private RpcClient client; private String hostname; private int port; private static Properties p= new Properties(); static { p.put("client.type","default"); p.put("hosts","h1"); p.put("hosts.h1","0.0.0.0:41414"); p.put("batch-size",100); p.put("connect-timeout",20000); p.put("request-timeout",20000); } public void init(String hostname, int port) { // Setup the RPC connection this.hostname = hostname; this.port = port; this.client = RpcClientFactory.getInstance(p); if (this.client == null) { System.out.println("init client fail"); } // this.client = RpcClientFactory.getInstance(hostname, port); // Use the following method to create a thrift client (instead of the above line): // this.client = RpcClientFactory.getThriftInstance(hostname, port); } public void sendDataToFlume(String data) { // Create a Flume Event object that encapsulates the sample data Event event = EventBuilder.withBody(data, Charset.forName("UTF-8")); // Send the event try { client.append(event); } catch (EventDeliveryException e) { // clean up and recreate the client client.close(); client = null; client = RpcClientFactory.getDefaultInstance(hostname, port); // Use the following method to create a thrift client (instead of the above line): // this.client = RpcClientFactory.getThriftInstance(hostname, port); e.printStackTrace(); } } public void cleanUp() { // Close the RPC connection client.close(); } }这个直接从官网copy下来的,唯一不同的地方是属性通过Properties类构建(后期方便抽出来作为一个配置文件),异常的堆栈需要打印出来,否则出错了都不带提示了,让新手很茫然(貌似成功了,但是agent那边收不到消息,其实是出错了)。
就这个简单的代码,在avro的jar包的版本未解决之前,报开头列出的错误,让人很无语。