Flume案例

2024-04-09 12:00:00 INFO: Application started
2024-04-09 12:05:00 ERROR: Something went wrong

定义Agent

agent.sources = logSource
agent.channels = memoryChannel
agent.sinks = hdfsSink

 配置Source agent

agent.sources.logSource.type = exec
agent.sources.logSource.command = tail -F /path/to/example.log
agent.sources.logSource.channels = memoryChannel

 配置Channel agent

agent.channels.memoryChannel.type = memory
agent.channels.memoryChannel.capacity = 1000
agent.channels.memoryChannel.transactionCapacity = 100

配置Sink agent

agent.sinks.hdfsSink.type = hdfs
agent.sinks.hdfsSink.hdfs.path = /flume/logs
agent.sinks.hdfsSink.hdfs.filePrefix = log-
agent.sinks.hdfsSink.hdfs.fileSuffix = .log
agent.sinks.hdfsSink.hdfs.fileType = DataStream
agent.sinks.hdfsSink.channel = memoryChannel

$ bin/flume-ng agent --conf conf --conf-file flume-example.conf --name agent -Dflume.root.logger=INFO,console