flume 选择器

选择器干什么的?选择什么?只有多个channel的时候才会用到选择器.可以选择把source中的数据如何分配给各个channel.
复制选择器就是复制好几份,每个channel都来一份一样的.

多路复用选择器俺一定的规则划分,这个更实用点,毕竟没事搞那么多副本干啥.
这个也是依据header中的键值对来进行匹配,然后把单个source中内容根据header 中key的value的不同分发到不同的channel中去.

举个例子

[root@mypc01 flumeconf]# vim multiplexing.properties
#命名并关联
a1.sources = r1
a1.channels = c1 c2 c3
a1.sinks = s1 s2 s3
a1.sources.r1.channels = c1 c2 c3
a1.sinks.s1.channel = c1
a1.sinks.s2.channel = c2
a1.sinks.s3.channel = c3

#指定source的属性
a1.sources.r1.type=http
a1.sources.r1.port=10086
a1.sources.r1.bind = mypc01
a1.sources.r1.handler = org.apache.flume.source.http.JSONHandler
#指定多副路选择器
a1.sources.r1.selector.type=multiplexing
a1.sources.r1.selector.header=gf
a1.sources.r1.selector.mapping.libai=c1
a1.sources.r1.selector.mapping.dufu=c2
a1.sources.r1.selector.default = c3



#指定channel的属性
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100
a1.channels.c2.type = memory
a1.channels.c2.capacity = 1000
a1.channels.c2.transactionCapacity = 100
a1.channels.c3.type = memory
a1.channels.c3.capacity = 1000
a1.channels.c3.transactionCapacity = 100

#指定sink的属性
a1.sinks.s1.type = hdfs
a1.sinks.s1.hdfs.path = hdfs://mypc01/%{gf}
a1.sinks.s1.hdfs.useLocalTimeStamp = true
a1.sinks.s1.hdfs.filePrefix = Flume
a1.sinks.s1.hdfs.fileSuffix = .log
a1.sinks.s1.hdfs.rollInterval = 30
a1.sinks.s1.hdfs.rollSize = 0
a1.sinks.s1.hdfs.rollCount = 0
a1.sinks.s1.hdfs.round = true  
a1.sinks.s1.hdfs.roundValue = 2
a1.sinks.s1.hdfs.roundUnit = minute
a1.sinks.s1.hdfs.fileType = DataStream
a1.sinks.s1.hdfs.writeFormat = Text

a1.sinks.s2.type = hdfs
a1.sinks.s2.hdfs.path = hdfs://mypc01/%{gf}
a1.sinks.s2.hdfs.useLocalTimeStamp = true
a1.sinks.s2.hdfs.filePrefix = Flume
a1.sinks.s2.hdfs.fileSuffix = .log
a1.sinks.s2.hdfs.rollInterval = 30
a1.sinks.s2.hdfs.rollSize = 0
a1.sinks.s2.hdfs.rollCount = 0
a1.sinks.s2.hdfs.round = true  
a1.sinks.s2.hdfs.roundValue = 2
a1.sinks.s2.hdfs.roundUnit = minute
a1.sinks.s2.hdfs.fileType = DataStream
a1.sinks.s2.hdfs.writeFormat = Text

a1.sinks.s3.type = hdfs
a1.sinks.s3.hdfs.path = hdfs://mypc01/others
a1.sinks.s3.hdfs.useLocalTimeStamp = true
a1.sinks.s3.hdfs.filePrefix = Flume
a1.sinks.s3.hdfs.fileSuffix = .log
a1.sinks.s3.hdfs.rollInterval = 30
a1.sinks.s3.hdfs.rollSize = 0
a1.sinks.s3.hdfs.rollCount = 0
a1.sinks.s3.hdfs.round = true  
a1.sinks.s3.hdfs.roundValue = 2
a1.sinks.s3.hdfs.roundUnit = minute
a1.sinks.s3.hdfs.fileType = DataStream
a1.sinks.s3.hdfs.writeFormat = Text

模拟执行

[root@mypc01 ~]# curl -X POST -d "[{"headers":{"gf":"libai"},"body":"helloworld"}]" http://mypc01:10086
# 执行结果,建立了一个libai的目录
2020-10-27 19:46:23,610 (SinkRunner-PollingRunner-DefaultSinkProcessor) [INFO - org.apache.flume.sink.hdfs.BucketWriter.open(BucketWriter.java:251)] Creating hdfs://mypc01/libai/Flume.1603799182926.log.tmp


[root@mypc01 ~]# curl -X POST -d "[{"header":{"gf":"dufu"},"body":"hello dufu sir"}]" http://mypc01:10086

# 执行结果
2020-10-27 19:48:31,571 (SinkRunner-PollingRunner-DefaultSinkProcessor) [INFO - org.apache.flume.sink.hdfs.BucketWriter.open(BucketWriter.java:251)] Creating hdfs://mypc01/dufu/Flume.1603799311489.log.tmp

[root@mypc01 ~]# curl -X POST -d "[{"header":{"gf":"others"},"body":"others"}]" http://mypc01:10086

# 执行结果
2020-10-27 19:53:43,030 (SinkRunner-PollingRunner-DefaultSinkProcessor) [INFO - org.apache.flume.sink.hdfs.BucketWriter.open(BucketWriter.java:251)] Creating hdfs://mypc01/others/Flume.1603799622454.log.tmp

总之,就是解决单个source如何分配到不同的channel的问题.