拦截 数据_大数据之六类Flume拦截器配置

---

时间戳拦截器

Timestamp.conf

#1.定义agent名， source、channel、sink的名称a4.sources = r1a4.channels = c1a4.sinks = k1#2.具体定义sourcea4.sources.r1.type = spooldira4.sources.r1.spoolDir = /opt/module/flume-1.8.0/upload#定义拦截器，为文件最后添加时间戳a4.sources.r1.interceptors = i1a4.sources.r1.interceptors.i1.type = org.apache.flume.interceptor.TimestampInterceptor$Builder#具体定义channela4.channels.c1.type = memorya4.channels.c1.capacity = 10000a4.channels.c1.transactionCapacity = 100#具体定义sinka4.sinks.k1.type = hdfsa4.sinks.k1.hdfs.path = hdfs://bigdata111:9000/flume-interceptors/%Ha4.sinks.k1.hdfs.filePrefix = events-a4.sinks.k1.hdfs.fileType = DataStream#不按照条数生成文件a4.sinks.k1.hdfs.rollCount = 0#HDFS上的文件达到128M时生成一个文件a4.sinks.k1.hdfs.rollSize = 134217728#HDFS上的文件达到60秒生成一个文件a4.sinks.k1.hdfs.rollInterval = 60#组装source、channel、sinka4.sources.r1.channels = c1a4.sinks.k1.channel = c1

主机名拦截器

#1.定义agenta1.sources= r1a1.sinks = k1a1.channels = c1#2.定义sourcea1.sources.r1.type = execa1.sources.r1.channels = c1a1.sources.r1.command = tail -F /opt/Andy#拦截器a1.sources.r1.interceptors = i1a1.sources.r1.interceptors.i1.type = host#参数为true时用IP192.168.1.111，参数为false时用主机名，默认为truea1.sources.r1.interceptors.i1.useIP = falsea1.sources.r1.interceptors.i1.hostHeader = agentHost#3.定义sinksa1.sinks.k1.type=hdfsa1.sinks.k1.channel = c1a1.sinks.k1.hdfs.path = hdfs://bigdata111:9000/flumehost/%Ha1.sinks.k1.hdfs.filePrefix = Andy_%{agentHost}#往生成的文件加后缀名.loga1.sinks.k1.hdfs.fileSuffix = .loga1.sinks.k1.hdfs.fileType = DataStreama1.sinks.k1.hdfs.writeFormat = Texta1.sinks.k1.hdfs.rollInterval = 10a1.sinks.k1.hdfs.useLocalTimeStamp = truea1.channels.c1.type = memorya1.channels.c1.capacity = 1000a1.channels.c1.transactionCapacity = 100a1.sources.r1.channels = c1a1.sinks.k1.channel = c1

UUID拦截器

a1.sources = r1a1.sinks = k1a1.channels = c1a1.sources.r1.type = execa1.sources.r1.channels = c1a1.sources.r1.command = tail -F /opt/Andya1.sources.r1.interceptors = i1#type的参数不能写成uuid，得写具体，否则找不到类a1.sources.r1.interceptors.i1.type = org.apache.flume.sink.solr.morphline.UUIDInterceptor$Builder#如果UUID头已经存在,它应该保存a1.sources.r1.interceptors.i1.preserveExisting = truea1.sources.r1.interceptors.i1.prefix = UUID_#如果sink类型改为HDFS，那么在HDFS的文本中没有headers的信息数据a1.sinks.k1.type = loggera1.channels.c1.type = memorya1.channels.c1.capacity = 1000a1.channels.c1.transactionCapacity = 100a1.sources.r1.channels = c1a1.sinks.k1.channel = c1# bin/flume-ng agent -c conf/ -f jobconf/uuid.conf -n a1 -Dflume.root.logger==INFO,console

查询替换拦截器

#1 agenta1.sources = r1a1.sinks = k1a1.channels = c1#2 sourcea1.sources.r1.type = execa1.sources.r1.channels = c1a1.sources.r1.command = tail -F /opt/Andya1.sources.r1.interceptors = i1a1.sources.r1.interceptors.i1.type = search_replace#遇到数字改成itstar，A123会替换为Aitstara1.sources.r1.interceptors.i1.searchPattern = [0-9]+a1.sources.r1.interceptors.i1.replaceString = itstara1.sources.r1.interceptors.i1.charset = UTF-8#3 sinka1.sinks.k1.type = logger#4 Chanela1.channels.c1.type = memorya1.channels.c1.capacity = 1000a1.channels.c1.transactionCapacity = 100#5 binda1.sources.r1.channels = c1a1.sinks.k1.channel = c1# bin/flume-ng agent -c conf/ -f jobconf/search.conf -n a1 -Dflume.root.logger=INFO,console

正则过滤拦截器

#1 agenta1.sources = r1a1.sinks = k1a1.channels = c1#2 sourcea1.sources.r1.type = execa1.sources.r1.channels = c1a1.sources.r1.command = tail -F /opt/Andya1.sources.r1.interceptors = i1a1.sources.r1.interceptors.i1.type = regex_filtera1.sources.r1.interceptors.i1.regex = ^A.*#如果excludeEvents设为false,表示过滤掉不是以A开头的events。如果excludeEvents设为true，则表示过滤掉以A开头的events。a1.sources.r1.interceptors.i1.excludeEvents = truea1.sinks.k1.type = loggera1.channels.c1.type = memorya1.channels.c1.capacity = 1000a1.channels.c1.transactionCapacity = 100a1.sources.r1.channels = c1a1.sinks.k1.channel = c1# bin/flume-ng agent -c conf/ -f jobconf/filter.conf -n a1 -Dflume.root.logger=INFO,console

正则抽取拦截器

#1 agenta1.sources = r1a1.sinks = k1a1.channels = c1#2 sourcea1.sources.r1.type = execa1.sources.r1.channels = c1a1.sources.r1.command = tail -F /opt/Andya1.sources.r1.interceptors = i1a1.sources.r1.interceptors.i1.type = regex_extractora1.sources.r1.interceptors.i1.regex = hostname is (.*?) ip is (.*)a1.sources.r1.interceptors.i1.serializers = s1 s2a1.sources.r1.interceptors.i1.serializers.s1.name = hostnamea1.sources.r1.interceptors.i1.serializers.s2.name = ipa1.sinks.k1.type = loggera1.channels.c1.type = memorya1.channels.c1.capacity = 1000a1.channels.c1.transactionCapacity = 100a1.sources.r1.channels = c1a1.sinks.k1.channel = c1# bin/flume-ng agent -c conf/ -f jobconf/extractor.conf -n a1 -Dflume.root.logger=INFO,console