Flume： ------ Sink Processors实现Sink Group的负载均衡、优先级选择

Apache Flume

flume官方用户指南网址

启动flume

[root@Centos ~]#  cd /usr/apache-flume-1.9.0-bin/
[root@Centos apache-flume-1.9.0-bin]#  ./bin/flume-ng version
Flume 1.9.0
Source code repository: https://git-wip-us.apache.org/repos/asf/flume.git
Revision: d4fcab4f501d41597bc616921329a4339f73585e
Compiled by fszabo on Mon Dec 17 20:45:25 CET 2018
From source with checksum 35db629a3bda49d23e9b3690c80737f9

Sink Processors实现负载均衡

• Load balancing Sink Processor 负载平衡接收器处理器

创建Flume文本文件example16.properties

[root@Centos apache-flume-1.9.0-bin]# vim conf/example16.properties

# 声明基本组件 Source Channel Sink example16.properties
a1.sources = s1
a1.sinks = sk1 sk2
a1.channels = c1
# 配置Source组件,从Socket中接收⽂文本数据
a1.sources.s1.type = netcat
a1.sources.s1.bind = Centos
a1.sources.s1.port = 44444
# 配置Sink组件,将接收数据打印在⽇日志控制台
a1.sinks.sk1.type = file_roll
a1.sinks.sk1.sink.directory = /root/baizhi/file_roll_1
a1.sinks.sk1.sink.rollInterval = 0
a1.sinks.sk1.sink.batchSize = 1
a1.sinks.sk2.type = file_roll
a1.sinks.sk2.sink.directory = /root/baizhi/file_roll_2
a1.sinks.sk2.sink.rollInterval = 0
a1.sinks.sk2.sink.batchSize = 1
# 配置Sink Porcessors
a1.sinkgroups = g1
a1.sinkgroups.g1.sinks = sk1 sk2
# 负载均衡
a1.sinkgroups.g1.processor.type = load_balance
a1.sinkgroups.g1.processor.backoff = true
# 轮训
a1.sinkgroups.g1.processor.selector = round_robin
# 配置Channel通道，主要负责数据缓冲
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
# 想要做出轮训效果必须设置为1
a1.channels.c1.transactionCapacity = 1
# 进⾏行行组件间的绑定
a1.sources.s1.channels = c1
a1.sinks.sk1.channel = c1
a1.sinks.sk2.channel = c1

• 如果想看到负载均衡效果， sink.batchSize 和 transactionCapacity 必须配置成1

启动Flume端

[root@Centos apache-flume-1.9.0-bin]# ./bin/flume-ng agent --conf conf/ --name a1 --conf-file conf/example16.properties  -Dflume.root.logger=INFO,console

telnet输入端

[root@Centos ~]# telnet Centos 44444
Trying 192.168.17.150...
Connected to Centos.
Escape character is '^]'.
a
OK
b
OK
c
OK
d
OK
e
OK
f
OK

结果

• 观察file_roll_1/file_roll_2中输出发现是轮训的

Sink Processors实现Sink Group的优先级

• Failover Sink Processor 故障转移接收器处理器

创建Flume文本文件example17.properties

[root@Centos apache-flume-1.9.0-bin]# vim conf/example17.properties

# 声明基本组件 Source Channel Sink example17.properties
a1.sources = s1
a1.sinks = sk1 sk2
a1.channels = c1
# 配置Source组件,从Socket中接收⽂文本数据
a1.sources.s1.type = netcat
a1.sources.s1.bind = Centos
a1.sources.s1.port = 44444
# 配置Sink组件,将接收数据打印在⽇日志控制台
a1.sinks.sk1.type = file_roll
a1.sinks.sk1.sink.directory = /root/baizhi/file_roll_1
a1.sinks.sk1.sink.rollInterval = 0
a1.sinks.sk1.sink.batchSize = 1
a1.sinks.sk2.type = file_roll
a1.sinks.sk2.sink.directory = /root/baizhi/file_roll_2
a1.sinks.sk2.sink.rollInterval = 0
a1.sinks.sk2.sink.batchSize = 1
# 配置Sink Porcessors
a1.sinkgroups = g1
a1.sinkgroups.g1.sinks = sk1 sk2
a1.sinkgroups.g1.processor.type = failover
# 设置优先级 SK1大于SK2
a1.sinkgroups.g1.processor.priority.sk1 = 20
a1.sinkgroups.g1.processor.priority.sk2 = 10
a1.sinkgroups.g1.processor.maxpenalty = 10000
# 配置Channel通道，主要负责数据缓冲
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 1
# 进⾏行行组件间的绑定
a1.sources.s1.channels = c1
a1.sinks.sk1.channel = c1
a1.sinks.sk2.channel = c1

telnet输入数据端

[root@Centos ~]# telnet Centos 44444
Trying 192.168.17.150...
Connected to Centos.
Escape character is '^]'.
A
OK
B
OK
C
OK
D
OK
E
OK

结果

• 只有file_roll_1中有数据

再试试如下

• 设置优先级 SK2大于SK1

# 设置优先级 SK2大于SK1
a1.sinkgroups.g1.processor.priority.sk1 = 20
a1.sinkgroups.g1.processor.priority.sk2 = 30

结果

• 只有file_roll_2中有数据