Flume-ng生产环境实践（四）实现log格式化interceptor

续上篇，由于filesink中需要使用/data/log/%{dayStr}/log-%{hourStr}%{minStr}-这样文件格式的，为了使file-sink能使用%{dayStr}这样的标签，需要在数据传输过程中，给event的header中添加对应的键值对。在flume-ng中提供了很方便的方式： Interceptor

以下为实现的interceptor，首先使用正则表达式匹配nginx日志，如何匹配成功，则获取匹配到的数据，并且对url中的参数进行处理，最后所有日志信息都被存储在Map中。根据配置文件中需要输出的键找到对应的值，按照顺序输出为csv格式的行。

原始日志格式：

112.245.239.72 - - [29/Dec/2012:15:00:00 +0800] "GET /p.gif?a=1&b=2 HTTP/1.1" 200 0 "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4 .0; 4399Box.1357; .NET CLR 2.0.50727; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; AskTbPTV2/5.9.1.14019; 4399Box.1357)"

最终结果：

1,2

配置信息为：

agent.sources = source agent.channels = channel agent.sinks = sink agent.sources.source.type = exec #agent.sources.source.command = tail -n +0 -F /data/tmp/accesspvpb_2012-11-18.log agent.sources.source.command = cat /opt/nginx/logs/vvaccess_log_pipe agent.sources.source.interceptors = logformat agent.sources.source.interceptors.logformat.type = org.apache.flume.interceptor.LogFormatInterceptor$Builder agent.sources.source.interceptors.logformat.confpath = /usr/programs/flume/conf/logformat_vv.properties agent.sources.source.interceptors.logformat.dynamicprop = true agent.sources.source.interceptors.logformat.hostname = vv111 agent.sources.source.interceptors.logformat.prop.monitor.rollInterval = 100000 # The channel can be defined as follows. agent.sources.source.channels = channel agent.sinks.sink.type = avro agent.sinks.sink.hostname = 192.168.0.100 agent.sinks.sink.port = 44444 agent.sinks.sink.channel = channel # Each channel's type is defined. agent.channels.channel.type = file agent.channels.channel.checkpointDir = /data/tmpc/checkpoint agent.channels.channel.dataDirs = /data/tmpc/data agent.channels.channel.transactionCapacity = 15000

/usr/programs/flume/conf/logformat_vv.properties文件内容为：

keys=a,b regexp=([0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3})\\s-\\s-\\s\\[([^]]+)\\]\\s\"GET\\s/p.gif\\?(.+)\\s.*\"\\s[0-9]+\\s[0-9]+\\s\"(.+)\"

interceptor的代码：

package  org.apache.flume.interceptor; import  static  org.apache.flume.interceptor.LogFormatInterceptor.Constants. CONF_PATH ; import  static  org.apache.flume.interceptor.LogFormatInterceptor.Constants. DYNAMICPROP ; import  static  org.apache.flume.interceptor.LogFormatInterceptor.Constants. DYNAMICPROP_DFLT ; import  static  org.apache.flume.interceptor.LogFormatInterceptor.Constants. HOSTNAME ; import  static  org.apache.flume.interceptor.LogFormatInterceptor.Constants. HOSTNAME_DFLT ; import  static  org.apache.flume.interceptor.LogFormatInterceptor.Constants. PROPMONITORINTERVAL ; import  static  org.apache.flume.interceptor.LogFormatInterceptor.Constants. PROPMONITORINTERVAL_DFLT ; import  java.io.File; import  java.io.FileInputStream; import  java.io.FileNotFoundException; import  java.io.IOException; import  java.text.ParseException; import  java.text.SimpleDateFormat; import  java.util.Date; import  java.util.HashMap; import  java.util.LinkedList; import  java.util.List; import  java.util.Map; import  java.util.Properties; import  org.apache.flume.Context; import  org.apache.flume.Event; import  org.apache.flume.event.EventBuilder; import  org.apache.oro.text.regex.MalformedPatternException; import  org.apache.oro.text.regex.MatchResult; import  org.apache.oro.text.regex.Pattern; import  org.apache.oro.text.regex.PatternCompiler; import  org.apache.oro.text.regex.PatternMatcher; import  org.apache.oro.text.regex.Perl5Compiler; import  org.apache.oro.text.regex.Perl5Matcher; import  org.slf4j.Logger; import  org.slf4j.LoggerFactory; public  class  LogFormatInterceptor  implements  Interceptor {          private  static  final  Logger  logger  = LoggerFactory                      . getLogger(LogFormatInterceptor. class );          private  String  conf_path  =  null ;          private  boolean  dynamicProp  =  false ;          private  String  hostname  =  null ;          private  long  propLastModify  = 0;          private  long  propMonitorInterval  ;          private  String  regexp  =  null ;          private  List<String>  keys  =  null ;          private  Pattern  pattern  =  null ;          private  PatternCompiler  compiler  =  null ;          private  PatternMatcher  matcher  =  null ;          private  SimpleDateFormat  sdf  =  null ;          private  SimpleDateFormat  sd  =  null ;          private  SimpleDateFormat  sh  =  null ;          private  SimpleDateFormat  sm  =  null ;          private  SimpleDateFormat  sdfAll  =  null ;          private  long  eventCount  = 0l;          public  LogFormatInterceptor(String conf_path,  boolean  dynamicProp,                      String hostname,  long  propMonitorInterval) {                 this . conf_path  = conf_path;                 this . dynamicProp  = dynamicProp;                 this . hostname  = hostname;                 this . propMonitorInterval  = propMonitorInterval;        }          @Override          public  void  close () {        }          @Override          public  void  initialize () {                 try  {                        // 读取配置文件，初始化正在表达式和输出的key列表                      File file =  new  File( conf_path  );                        propLastModify  = file.lastModified();                      Properties props =  new  Properties();                      FileInputStream fis;                      fis =  new  FileInputStream(file);                      props.load(fis);                        regexp  = props.getProperty(  "regexp" );                      String strKey = props.getProperty(  "keys" );                        if  (strKey !=  null ) {                            String[] strkeys = strKey.split(  "," );                              keys  =  new  LinkedList<String>();                              for  (String key : strkeys) {                                     keys .add(key);                            }                      }                        if  ( keys  ==  null ) {                              logger .error( "====================keys is null====================" );                      }  else  {                              logger .info( "keys="  +  keys  );                      }                        if  ( regexp  ==  null ) {                              logger .error( "====================regexp is null====================" );                      }  else  {                              logger .info( "regexp="  +  regexp  );                      }                        // 初始化正在表达式以及时间格式化类                        compiler  =  new  Perl5Compiler();                        pattern  =  compiler .compile(  regexp );                        matcher  =  new  Perl5Matcher();                        sdf  =  new  SimpleDateFormat( "dd/MMM/yyyy:HH:mm:ss Z"  ,                                   java.util.Locale.  US );                        sd  =  new  SimpleDateFormat( "yyyyMMdd"  );                        sh  =  new  SimpleDateFormat( "HH"  );                        sm  =  new  SimpleDateFormat( "mm"  );                        sdfAll  =  new  SimpleDateFormat( "yyyyMMddHHmmss"  );               }  catch  (MalformedPatternException e) {                        logger .error( "Could not complile pattern!"  , e);               }  catch  (FileNotFoundException e) {                        logger .error( "conf file is not found!"  , e);               }  catch  (IOException e) {                        logger .error( "conf file can not be read!"  , e);               }        }          @Override          public  Event intercept(Event event) {               ++  eventCount ;                 try  {                        if  ( dynamicProp  &&  eventCount  >  propMonitorInterval ) {                            File file =  new  File( conf_path  );                              if  (file.lastModified() >  propLastModify  ) {                                     propLastModify  = file.lastModified();                                   Properties props =  new  Properties();                                   FileInputStream fis;                                   fis =  new  FileInputStream(file);                                   props.load(fis);                                   String strKey = props.getProperty(  "keys" );                                     if  (strKey !=  null ) {                                          String[] strkeys = strKey.split( ","  );                                          List<String> keystmp =  new  LinkedList<String>();                                            for  (String key : strkeys) {                                                 keystmp.add(key);                                          }                                            if  (keystmp.size() >  keys  .size()) {                                                   keys  = keystmp;                                                   logger .info( "dynamicProp status updated = "  +  keys );                                          }  else  {                                                   logger .error( "dynamicProp status new keys size less than old,so status update fail = "                                                               +  keys );                                          }                                   }  else  {                                            logger .error( "dynamicProp status get keys fail ,so status update fail = "                                                        +  keys );                                   }                            }                      }                      Map<String, String> headers = event.getHeaders();                      headers.put(  "host" ,  hostname  );                      String body =  new  String(event.getBody());                        if  ( pattern  !=  null ) {                            StringBuffer stringBuffer =  new  StringBuffer();                            Date date =  null ;                            Map<String, String> index =  new  HashMap<String, String>();                              if  ( matcher  .contains(body,  pattern )) {                                   index.put(  "host" ,  hostname  );                                   MatchResult result =  matcher .getMatch();                                   index.put(  "ip" , result.group(1));                                     try  {                                          date =  sdf .parse(result.group(2));                                          index.put(  "loc_time" ,  sdfAll .format(date));                                   }  catch  (ParseException e1) {                                   }                                   String url = result.group(3).replaceAll(  "," ,  "|" );                                   String[] params = url.split(  "&" );                                     for  (String param : params) {                                          String[] p = param.split( "="  );                                            if  (p. length  == 2) {                                                 index.put(p[0], p[1]);                                          }                                   }                                   index.put(  "browser" , result.group(4).replaceAll( ","  ,  "|"  ));                                     for  (String key :  keys  ) {                                            if  (index.containsKey(key)) {                                                 stringBuffer.append(index.get(key) +  "," );                                          }  else  {                                                 stringBuffer.append(  "~," );                                          }                                   }                                     if  (stringBuffer.length() > 0) {                                          stringBuffer.deleteCharAt(stringBuffer.length() - 1);                                   }  else  {                                          stringBuffer.append(  "error="  + body);                                   }                                     if  (date !=  null ) {                                          headers.put(  "dayStr" ,  sd  .format(date));                                          headers.put(  "hourStr" ,  sh  .format(date));                                          Integer m = Integer.parseInt( sm .format(date));                                          String min =  "" ;                                            if  (m >= 0 && m < 10) {                                                 min =  "0"  + (m / 5) * 5;                                          }  else  {                                                 min = (m / 5) * 5 +  ""  ;                                          }                                          headers.put(  "minStr" , min);                                   }  else  {                                          headers.put(  "dayStr" ,  "errorLog"  );                                   }                                   Event e = EventBuilder.withBody(stringBuffer.toString()                                                 .getBytes(), headers);                                     return  e;                            }                      }               }  catch  (Exception e) {                        logger .error( "LogFormat error!"  , e);               }                 return  null  ;        }          @Override          public  List<Event> intercept(List<Event> events) {               List<Event> list =  new  LinkedList<Event>();                 for  (Event event : events) {                      Event e = intercept(event);                        if  (e !=  null  ) {                            list.add(e);                      }               }                 return  list;        }          /**         * Builder which builds new instances of the HostInterceptor.         */          public  static  class  Builder  implements  Interceptor.Builder {                 private  String  confPath  ;                 private  boolean  dynamicProp  ;                 private  String  hostname  ;                 private  long  propMonitorInterval  ;                 @Override                 public  Interceptor build() {                        return  new  LogFormatInterceptor( confPath ,  dynamicProp ,  hostname ,                                     propMonitorInterval );               }                 @Override                 public  void  configure(Context context) {                        confPath  = context.getString(  CONF_PATH );                        dynamicProp  = context.getBoolean( DYNAMICPROP ,  DYNAMICPROP_DFLT );                        hostname  = context.getString(  HOSTNAME ,  HOSTNAME_DFLT  );                        propMonitorInterval  = context.getLong( PROPMONITORINTERVAL ,                                     PROPMONITORINTERVAL_DFLT );               }        }          public  static  class  Constants {                 public  static  String  CONF_PATH  =  "confpath" ;                 public  static  String  DYNAMICPROP  =  "dynamicprop" ;                 public  static  boolean  DYNAMICPROP_DFLT  =  false ;                 public  static  String  HOSTNAME  =  "hostname" ;                 public  static  String  HOSTNAME_DFLT  =  "hostname" ;                 public  static  String  PROPMONITORINTERVAL  =  "prop.monitor.rollInterval"  ;                 public  static  long  PROPMONITORINTERVAL_DFLT  = 500000l;        } }

至此，获取nginx日志，进行格式化清洗，传输到collector机器，按照格式化的目录和文件名进行输出全部完成。