Flume-ng生产环境实践（三）实现文件sink，按照固定格式目录输出

应用场景：需要实时收集多台服务器的nginx日志到一台机器。收集完成结果存放需要按天生成文件夹，按每5分钟生成文件，比如2012年12月29日12点26分的日志，需要放到/data/log/20121229/log-1225-对应的文件中。自己实现了类似flume-og和flume-ng的hdfs-sink的文件sink。

使用的时候配置如下：

agent.sources = source agent.channels = channel agent.sinks = sink agent.sources.source.type = avro agent.sources.source.bind = 192.168.0.100 agent.sources.source.port = 44444 agent.sources.source.channels = channel agent.sinks.sink.type = org.apache.flume.sink.FileSink agent.sinks.sink.file.path = /data/log/%{dayStr} agent.sinks.sink.file.filePrefix = log-%{hourStr}%{minStr}- agent.sinks.sink.file.txnEventMax = 10000 agent.sinks.sink.file.maxOpenFiles = 5 agent.sinks.sink.channel = channel agent.channels.channel.type = memory agent.channels.channel.capacity = 100000 agent.channels.channel.transactionCapacity = 100000 agent.channels.channel.keep-alive = 60

依赖的jar如下：

jakarta-oro-2.0.1.jar

flume-ng-core-1.3.0-SNAPSHOT.jar

flume-ng-sdk-1.3.0-SNAPSHOT.jar

flume-ng-configuration-1.3.0-SNAPSHOT.jar

slf4j-log4j12-1.6.1.jar

slf4j-api-1.6.1.jar

guava-10.0.1.jar

代码如下：

FileSink.java

package  org.apache.flume.sink; import  java.io.IOException; import  java.util.Calendar; import  java.util.List; import  java.util.concurrent.Executors; import  java.util.concurrent.ScheduledExecutorService; import  org.apache.flume.Channel; import  org.apache.flume.Context; import  org.apache.flume.Event; import  org.apache.flume.EventDeliveryException; import  org.apache.flume.Transaction; import  org.apache.flume.conf.Configurable; import  org.apache.flume.formatter.output.BucketPath; import  org.apache.flume.instrumentation.SinkCounter; import  org.apache.flume.serialization.EventSerializer; import  org.slf4j.Logger; import  org.slf4j.LoggerFactory; import  com.google.common.base.Preconditions; import  com.google.common.collect.Lists; import  com.google.common.util.concurrent.ThreadFactoryBuilder; public  class  FileSink  extends  AbstractSink  implements  Configurable {          private  static  final  Logger  logger  = LoggerFactory                      . getLogger(FileSink . class  );          private  String  path  ;          private  static  final  String  defaultFileName  =  "FlumeData" ;          private  static  final  int  defaultMaxOpenFiles  = 50;          /**         * Default length of time we wait for blocking BucketWriter calls before         * timing out the operation. Intended to prevent server hangs.         */          private  long  txnEventMax  ;          private  FileWriterLinkedHashMap  sfWriters  ;          private  String  serializerType  ;          private  Context  serializerContext  ;          private  boolean  needRounding  =  false ;          private  int  roundUnit  = Calendar. SECOND ;          private  int  roundValue  = 1;          private  SinkCounter  sinkCounter  ;          private  int  maxOpenFiles  ;          private  ScheduledExecutorService  timedRollerPool  ;          private  long  rollInterval  ;          @Override          public  void  configure(Context context) {               String directory = Preconditions. checkNotNull(                            context.getString(  "file.path" ),  "file.path is required" );               String fileName = context.getString(  "file.filePrefix" ,  defaultFileName );                 this . path  = directory +  "/"  + fileName;                 maxOpenFiles  = context.getInteger( "file.maxOpenFiles"  ,                              defaultMaxOpenFiles );                 serializerType  = context.getString( "sink.serializer"  ,  "TEXT"  );                 serializerContext  =  new  Context(                            context.getSubProperties(EventSerializer.  CTX_PREFIX ));                 txnEventMax  = context.getLong( "file.txnEventMax"  , 1l);                 if  ( sinkCounter  ==  null ) {                        sinkCounter  =  new  SinkCounter(getName());               }                 rollInterval  = context.getLong( "file.rollInterval"  , 30l);               String rollerName =  "hdfs-"  + getName() +  "-roll-timer-%d"  ;                 timedRollerPool  = Executors.newScheduledThreadPool(  maxOpenFiles ,                              new  ThreadFactoryBuilder().setNameFormat(rollerName).build());        }          @Override          public  Status process()  throws  EventDeliveryException {               Channel channel = getChannel();               Transaction transaction = channel.getTransaction();               List<BucketFileWriter> writers = Lists. newArrayList();               transaction.begin();                 try  {                      Event event =  null ;                        int  txnEventCount = 0;                        for  (txnEventCount = 0; txnEventCount <  txnEventMax ; txnEventCount++) {                            event = channel.take();                              if  (event ==  null ) {                                     break ;                            }                              // reconstruct the path name by substituting place holders                            String realPath = BucketPath                                          . escapeString( path , event.getHeaders(),  needRounding ,                                                          roundUnit ,  roundValue  );                            BucketFileWriter bucketFileWriter =  sfWriters .get(realPath);                              // we haven't seen this file yet, so open it and cache the                              // handle                              if  (bucketFileWriter ==  null ) {                                   bucketFileWriter =  new  BucketFileWriter();                                   bucketFileWriter.open(realPath,  serializerType ,                                                   serializerContext ,  rollInterval  ,  timedRollerPool ,                                                   sfWriters );                                     sfWriters .put(realPath, bucketFileWriter);                            }                              // track the buckets getting written in this transaction                              if  (!writers.contains(bucketFileWriter)) {                                   writers.add(bucketFileWriter);                            }                              // Write the data to File                            bucketFileWriter.append(event);                      }                        if  (txnEventCount == 0) {                              sinkCounter .incrementBatchEmptyCount();                      }  else  if  (txnEventCount ==  txnEventMax ) {                              sinkCounter .incrementBatchCompleteCount();                      }  else  {                              sinkCounter .incrementBatchUnderflowCount();                      }                        // flush all pending buckets before committing the transaction                        for  (BucketFileWriter bucketFileWriter : writers) {                              if  (!bucketFileWriter.isBatchComplete()) {                                   flush(bucketFileWriter);                            }                      }                      transaction.commit();                        if  (txnEventCount > 0) {                               sinkCounter .addToEventDrainSuccessCount(txnEventCount);                      }                        if  (event ==  null ) {                              return  Status. BACKOFF  ;                      }                        return  Status. READY  ;               }  catch  (IOException eIO) {                      transaction.rollback();                        logger .warn( "File IO error"  , eIO);                        return  Status. BACKOFF  ;               }  catch  (Throwable th) {                      transaction.rollback();                        logger .error( "process failed"  , th);                        if  (th  instanceof  Error) {                              throw  (Error) th;                      }  else  {                              throw  new  EventDeliveryException(th);                      }               }  finally  {                      transaction.close();               }        }          private  void  flush(BucketFileWriter bucketFileWriter)  throws  IOException {               bucketFileWriter.flush();        }          @Override          public  synchronized  void  start() {                 super .start();                 this . sfWriters  =  new  FileWriterLinkedHashMap( maxOpenFiles );                 sinkCounter .start();        } }

BucketFileWriter.java

package org.apache.flume.sink; import java.io.BufferedOutputStream; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; import java.util.concurrent.Callable; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import org.apache.flume.Context; import org.apache.flume.Event; import org.apache.flume.serialization.EventSerializer; import org.apache.flume.serialization.EventSerializerFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class BucketFileWriter {      private static final Logger logger = LoggerFactory                .getLogger(BucketFileWriter.class);      private static final String IN_USE_EXT = ".tmp";      /**      * This lock ensures that only one thread can open a file at a time.      */      private final AtomicLong fileExtensionCounter;      private OutputStream outputStream;      private EventSerializer serializer;      private String filePath;      /**      * Close the file handle and rename the temp file to the permanent filename.      * Safe to call multiple times. Logs HDFSWriter.close() exceptions.      *      * @throws IOException      *             On failure to rename if temp file exists.      */      public BucketFileWriter() {           fileExtensionCounter = new AtomicLong(System.currentTimeMillis());      }      public void open(final String filePath, String serializerType,                Context serializerContext, final long rollInterval,                final ScheduledExecutorService timedRollerPool,                final FileWriterLinkedHashMap sfWriters) throws IOException {           this.filePath = filePath;           File file = new File(filePath + fileExtensionCounter + IN_USE_EXT);           file.getParentFile().mkdirs();           outputStream = new BufferedOutputStream(new FileOutputStream(file));           logger.info("filename = " + file.getAbsolutePath());           serializer = EventSerializerFactory.getInstance(serializerType,                     serializerContext, outputStream);           serializer.afterCreate();           if (rollInterval > 0) {                Callable<Void> action = new Callable<Void>() {                     @Override                     public Void call() throws Exception {                          logger.debug(                                    "Rolling file ({}): Roll scheduled after {} sec elapsed.",                                    filePath + fileExtensionCounter + IN_USE_EXT,                                    rollInterval);                          if (sfWriters.containsKey(filePath)) {                               sfWriters.remove(filePath);                          }                          close();                          return null;                     }                };                timedRollerPool.schedule(action, rollInterval, TimeUnit.SECONDS);           }      }      public void append(Event event) throws IOException {           serializer.write(event);      }      public boolean isBatchComplete() {           return true;      }      public void flush() throws IOException {           serializer.flush();           outputStream.flush();      }      /**      * Rename bucketPath file from .tmp to permanent location.      */      private void renameBucket() {           File srcPath = new File(filePath + fileExtensionCounter + IN_USE_EXT);           File dstPath = new File(filePath + fileExtensionCounter);           if (srcPath.exists()) {                srcPath.renameTo(dstPath);                logger.info("Renaming " + srcPath + " to " + dstPath);           }      }      public synchronized void close() throws IOException, InterruptedException {           if (outputStream != null) {                outputStream.flush();                outputStream.close();           }           renameBucket();      } }

FileWriterLinkedHashMap.java

package org.apache.flume.sink; import java.io.IOException; import java.util.LinkedHashMap; import java.util.Map.Entry; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class FileWriterLinkedHashMap extends           LinkedHashMap<String, BucketFileWriter> {      private static final Logger logger = LoggerFactory                .getLogger(FileWriterLinkedHashMap.class);      private static final long serialVersionUID = -7860596835613215998L;      private final int maxOpenFiles;      public FileWriterLinkedHashMap(int maxOpenFiles) {           super(16, 0.75f, true); // stock initial capacity/load, access           this.maxOpenFiles = maxOpenFiles;      }      @Override      protected boolean removeEldestEntry(Entry<String, BucketFileWriter> eldest) {           if (size() > maxOpenFiles) {                // If we have more that max open files, then close the last one                // and                // return true                try {                     eldest.getValue().close();                } catch (IOException e) {                     logger.warn(eldest.getKey().toString(), e);                } catch (InterruptedException e) {                     logger.warn(eldest.getKey().toString(), e);                     Thread.currentThread().interrupt();                }                return true;           } else {                return false;           }      } }