flume配置参考

1 redis source,memory channel,json intercepter and file_roll sink配置例子

agent.sources = redis redis1
agent.channels = memory
agent.sinks = fileSink


agent.sources.redis.type = com.flumeng.plugins.source.RedisLPopSource
agent.sources.redis.host = 192.168.121.128
agent.sources.redis.port = 6379
agent.sources.redis.eventNamePrefix = event
agent.sources.redis.timeout = 3000
agent.sources.redis.charset = utf-8
agent.sources.redis.interceptors = i1
agent.sources.redis.interceptors.i1.type = com.flumeng.plugins.source.JSONInterceptor$Builder


agent.sources.redis1.type = com.flumeng.plugins.source.RedisLRangeSource
agent.sources.redis1.host = 192.168.121.128
agent.sources.redis1.port = 6379
agent.sources.redis1.eventNamePrefix = event
agent.sources.redis1.timeout = 3000
agent.sources.redis1.charset = utf-8
agent.sources.redis1.eventbatchSize = 2000
agent.sources.redis1.interceptors = i2
agent.sources.redis1.interceptors.i2.type = com.flumeng.plugins.source.JSONInterceptor$Builder


agent.channels.memory.type = memory
agent.channels.memory.capacity = 1000000
agent.channels.memory.transactionCapacity = 100000


agent.sinks.fileSink.type = file_roll
agent.sinks.fileSink.channel = memory
agent.sinks.fileSink.sink.directory = /mnt/hgfs/share/result/
agent.sinks.fileSink.sink.rollInterval = 0
agent.sinks.fileSink.batchSize=500


agent.sources.redis.channels = memory
agent.sources.redis1.channels = memory

2 redis source,memory channel,json intercepter and avro sink配置例子

agent.sources = redis redis1
agent.channels = memory
agent.sinks = fileSink


agent.sources.redis.type = com.flumeng.plugins.source.RedisLPopSource
agent.sources.redis.host = 192.168.121.128
agent.sources.redis.port = 6379
agent.sources.redis.eventNamePrefix = event
agent.sources.redis.timeout = 3000
agent.sources.redis.charset = utf-8
agent.sources.redis.interceptors = i1
agent.sources.redis.interceptors.i1.type = com.flumeng.plugins.source.JSONInterceptor$Builder


agent.sources.redis1.type = com.flumeng.plugins.source.RedisLRangeSource
agent.sources.redis1.host = 192.168.121.128
agent.sources.redis1.port = 6379
agent.sources.redis1.eventNamePrefix = event
agent.sources.redis1.timeout = 3000
agent.sources.redis1.charset = utf-8
agent.sources.redis1.eventbatchSize = 2000
agent.sources.redis1.interceptors = i2
agent.sources.redis1.interceptors.i2.type = com.flumeng.plugins.source.JSONInterceptor$Builder


agent.channels.memory.type = memory
agent.channels.memory.capacity = 1000000
agent.channels.memory.transactionCapacity = 100000


agent.sinks.fileSink.type = file_roll
agent.sinks.fileSink.channel = memory
agent.sinks.fileSink.sink.directory = /mnt/hgfs/share/result/
agent.sinks.fileSink.sink.rollInterval = 0
agent.sinks.fileSink.batchSize=500


agent.sources.redis.channels = memory
agent.sources.redis1.channels = memory

3 avro source--memery channel ---HDFS sink---interceptors---serializers

agent.sources = avroSource
agent.channels = memory
agent.sinks = hdfsSink


agent.sources.avroSource.type = avro
agent.sources.avroSource.bind = 0.0.0.0
agent.sources.avroSource.port = 4141


agent.sources.avroSource.interceptors = i1
agent.sources.avroSource.interceptors.i1.type=regex_extractor
agent.sources.avroSource.interceptors.i1.regex = (\\d\\d\\d\\d-\\d\\d-\\d\\d\\s\\d\\d:\\d\\d)
agent.sources.avroSource.interceptors.i1.serializers = s1
agent.sources.avroSource.interceptors.i1.serializers.s1.type = org.apache.flume.interceptor.RegexExtractorInterceptorMillisSerializer
agent.sources.avroSource.interceptors.i1.serializers.s1.name = timestamp
agent.sources.avroSource.interceptors.i1.serializers.s1.pattern = yyyy-MM-dd HH:mm


agent.channels.memory.type = memory
agent.channels.memory.capacity = 1000000
agent.channels.memory.transactionCapacity = 100000


agent.sinks.hdfsSink.type = hdfs
agent.sinks.hdfsSink.channel = memory
agent.sinks.hdfsSink.hdfs.path = hdfs://littleNameservice/test/outputAnalysis/y=%Y/m=%m/d=%d/h=%H
agent.sinks.hdfsSink.hdfs.rollInterval =0 
agent.sinks.hdfsSink.hdfs.rollSize =268435456
agent.sinks.hdfsSink.hdfs.rollCount =0
#agent.sinks.hdfsSink.hdfs.filePrefix =%Y_%m_%d_%H
agent.sinks.hdfsSink.hdfs.filePrefix =event
agent.sinks.hdfsSink.hdfs.fileSuffix =.txt
agent.sinks.hdfsSink.hdfs.inUseSuffix =.tmp
agent.sinks.hdfsSink.hdfs.batchSize = 10000
agent.sinks.hdfsSink.hdfs.idleTimeout = 600
agent.sinks.hdfsSink.hdfs.fileType = DataStream


agent.sources.avroSource.channels = memory

4 avro source---memery channel---hbase sink---serializer---interceptor

agent.sources = avroSource
agent.channels = memory
agent.sinks = hbaseSink


agent.sources.avroSource.type = avro
agent.sources.avroSource.bind = 0.0.0.0
agent.sources.avroSource.port = 4141


agent.sources.avroSource.interceptors = i1
agent.sources.avroSource.interceptors.i1.type=com.jj.flume.interceptor.json.EventHbaseInterceptor$Builder


agent.channels.memory.type = memory
agent.channels.memory.capacity = 1000000
agent.channels.memory.transactionCapacity = 100000


agent.sinks.hbaseSink.type = hbase
agent.sinks.hbaseSink.table = t_event
agent.sinks.hbaseSink.columnFamily = f_event
agent.sinks.hbaseSink.serializer = com.jj.flume.sink.hbase.RegexHbaseEventSerializerJJ
agent.sinks.hbaseSink.channel = memory


agent.sources.avroSource.channels = memory