http sink自定义

 

问题：httpsink到终端接口依然用flume 的httpsource收集数据发现拿不到数据。发现源码的httpsink之后只保留了body的value其他信息都没了这里自定义一下，也可将header封装回去。

package test;

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.google.common.base.Preconditions;
import com.google.common.base.Throwables;
import okhttp3.*;
import org.apache.flume.*;
import org.apache.flume.conf.Configurable;
import org.apache.flume.sink.AbstractSink;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

public class HttpSink extends AbstractSink implements Configurable {
   private static Logger LOG = LoggerFactory.getLogger(HttpSink.class);
   private String hostname;
   private String port;
   private String url;
   private int batchSize;
   private String postUrl;
   public HttpSink(){
      LOG.info("HttpSink start....");
   }

   @Override
   public void configure(Context context) {
      hostname = context.getString("hostname");
      Preconditions.checkNotNull("hostname","hostname must not be null!");
      port = context.getString("port");
      Preconditions.checkNotNull("port","port must not be null!");
      batchSize = context.getInteger("batchSize",100);
      Preconditions.checkArgument(batchSize>0,"batchSize must be a positive number!");
      url = context.getString("url");
      postUrl = "http://"+hostname+":"+port+url;
   }

   @Override
   public Status process() throws EventDeliveryException {
      Status result = Status.READY;
      Channel channel = getChannel();
      Transaction transaction = null;
      try {
         transaction = channel.getTransaction();
         transaction.begin();
         Event event = null;
         String content = null;

         List<Map<String,String>> contents = new ArrayList<>();
         for(int i = 0 ;i < batchSize;i++){
            Map<String,String> content1= new HashMap<>();
            event = channel.take();
            if(null != event){
               //对事件进行处理,源码对事件只取了body，如果，http sink的终端是flume采集的，会收集不到信息，这里再加上body，也可再加上其他的封装，header都可以通过event拿到。
               content = new String(event.getBody());
               if(null != content && !"".equals(content)){
                  content1.put("body",content);
                  contents.add(content1);
//                contents.add(JSONObject.toJSON(content1).toString());
               }
            }else {
               result = Status.BACKOFF;
               break;
            }
         }
         if(contents.size() > 0){
            Response response = postJson(postUrl, JSONArray.toJSON(contents).toString());
            if(null != response && response.isSuccessful()){
               //commit 机制确保数据并不会丢失
               transaction.commit();
            }
         }else {
            transaction.commit();
         }
      }catch (Exception e){
         try {
            if(null != transaction){
               transaction.rollback();
            }
         }catch (Exception e2){
            LOG.error("Exception in rollback. Rollback might not be successful." + e2);
         }
         LOG.error("Failed to commit transaction." + e);
         Throwables.propagate(e);

      }finally {
         if(null != transaction){
            transaction.close();
            LOG.debug("close Transaction");
         }
      }
      return result;
   }

   /**
    * post 请求，body为json数据
    * @param url
    * @param json
    * @return
    */
   private Response postJson(String url, String json){
      OkHttpClient client = new OkHttpClient();
      RequestBody body = RequestBody.create(MediaType.parse("application/json"),json);
      Request request = new Request.Builder()
            .url(url)
            .post(body)
            .build();
      Response response = null;
      try {
         response = client.newCall(request).execute();
         if(!response.isSuccessful()){
            LOG.info("request was error!");
         }
      }catch (IOException e){
         LOG.error("IOException in postJson." + e);
         e.printStackTrace();
      }

      return response;
   }

   @Override
   public synchronized void stop() {
      super.stop();
   }

   @Override
   public synchronized void start() {
      super.start();
   }
}