主因
在上篇博客中,我们已经做到了非常不错的多文件检索,但是,还有一个问题,就是针对不同的项目,需要多个配置文件,这样,对运维,是个非常繁琐的问题,针对这个问题,本版对flume扩展了文件组(以|切分父文件)。
cat监控,是个久经考验的监控报警平台,因此决定兼容cat的协议,对source进一步扩展。
针对很多人说方法过大的问题,本次也做了调整,优化了设计,及传值方式,详情见具体代码。
说到这,是不是很激动,因为我们随着时间的推进,我们在易维护,高可用的道路上走得越来越远,flume一定还可以优化,还待我们去挖掘!
cat结果:
源码
主干类:
/* * 作者:许恕 * 时间:2016年5月3日 * 功能:实现tail 某目录下的所有符合正则条件的文件 * Email:xvshu1@163.com * To detect all files in a folder */ package org.apache.flume.source; import com.dianping.cat.Cat; import com.dianping.cat.message.Transaction; import com.el.ump.profiler.Profiler; import com.google.common.base.Preconditions; import com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.flume.Context; import org.apache.flume.Event; import org.apache.flume.EventDrivenSource; import org.apache.flume.SystemClock; import org.apache.flume.channel.ChannelProcessor; import org.apache.flume.conf.Configurable; import org.apache.flume.event.EventBuilder; import org.apache.flume.instrumentation.SourceCounter; import org.apache.flume.source.utils.MsgBuildeJson; import org.mortbay.util.ajax.JSON; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.*; import java.nio.charset.Charset; import java.util.*; import java.util.concurrent.*; import java.util.regex.Matcher; import java.util.regex.Pattern; /** * step: * 1,config one path * 2,find all file with RegExp * 3,tail one children file * 4,batch to channal * * demo: * demo.sources.s1.type = org.apache.flume.source.ExecTailSource * demo.sources.s1.filepath=/export/home/tomcat/logs/auth.el.net/ * demo.sources.s1.filenameRegExp=(.log{1})$ * demo.sources.s1.tailing=true * demo.sources.s1.readinterval=300 * demo.sources.s1.startAtBeginning=false * demo.sources.s1.restart=true */ public class ExecTailSource extends AbstractSource implements EventDrivenSource, Configurable { private static final Logger logger = LoggerFactory .getLogger(ExecTailSource.class); private SourceCounter sourceCounter; private ExecutorService executor; private List<ExecRunnable> listRuners; private List<Future<?>> listFuture; private long restartThrottle; private boolean restart = true; private boolean logStderr; private Integer bufferCount; private long batchTimeout; private Charset charset; private String filepath; private String filenameRegExp; private boolean tailing; private Integer readinterval; private boolean startAtBeginning; private boolean contextIsJson; private String fileWriteJson; private Long flushTime; private boolean contextIsFlumeLog; private String domain; private String msgTypeConfig; @Override public void start() { logger.info("=start=> flume tail source start begin time:"+new Date().toString()); logger.info("ExecTail source starting with filepath:{}", filepath); List<String> listFiles = getFileList(filepath); if(listFiles==null || listFiles.isEmpty()){ Preconditions.checkState(listFiles != null && !listFiles.isEmpty(), "The filepath's file not have fiels with filenameRegExp"); } Properties prop=null; try{ prop = new Properties();//属性集合对象 FileInputStream fis = new FileInputStream(fileWriteJson);//属性文件流 prop.load(fis); }catch(Exception ex){ logger.error("==>",ex); } executor = Executors.newFixedThreadPool(listFiles.size()); listRuners = new ArrayList<ExecRunnable>(); listFuture = new ArrayList<Future<?>>(); logger.info("files size is {} ", listFiles.size()); // FIXME: Use a callback-like executor / future to signal us upon failure. for(String oneFilePath : listFiles){ ExecRunnable runner = new ExecRunnable(getChannelProcessor(), sourceCounter, restart, restartThrottle, logStderr, bufferCount, batchTimeout, charset,oneFilePath,tailing,readinterval,startAtBeginning,contextIsJson, prop,fileWriteJson,flushTime,contextIsFlumeLog,domain); listRuners.add(runner); Future<?> runnerFuture = executor.submit(runner); listFuture.add(runnerFuture); logger.info("{} is begin running",oneFilePath); } /* * NB: This comes at the end rather than the beginning of the method because * it sets our state to running. We want to make sure the executor is alive * and well first. */ sourceCounter.start(); super.start(); logger.info("=start=> flume tail source start end time:"+new Date().toString()); logger.debug("ExecTail source started"); } @