flume install

下载地址：
http://archive.apache.org/dist/flume/
我这里下载的是apache-flume-1.9.0-bin.tar.gz

mkdir -p /data/flume
将压缩包放入该目录
#解压
tar -zxvf apache-flume-1.9.0-bin.tar.gz

#修改配置文件
cp conf/flume-env.sh.template conf/flume-env.sh
export JAVA_HOME=/data/jdk/jdk1.8.0_271

#配置Flume Agent,侦听网络端口数据
cp conf/flume-conf.properties.template conf/flume-conf.properties

a1.sources = r1
a1.channels = c1
a1.sinks = k1

# For each one of the sources, the type is defined
a1.sources.r1.type = netcat
a1.sources.r1.bind = localhost
a1.sources.r1.port = 8888

# The channel can be defined as follows.
a1.sources.r1.channels = c1

# Each sink's type must be defined
a1.sinks.k1.type = logger

#Specify the channel the sink should use
a1.sinks.k1.channel = c1

# Each channel's type is defined.
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100

# Other config values specific to each type of channel(sink or source)
# can be defined as well
# In this case, it specifies the capacity of the memory channel
a1.channels.memoryChannel.capacity = 100


#启动flume agent
bin/flume-ng agent -c conf -n a1 -f conf/flume-conf.properties -Dflume.root.logger=INFO,console

安装nc
yum install nc -y


[root@yfm19 ~]# nc localhost 8888
hello
OK
nihao
OK
^C
此时可以查看flume agent启动终端的输出
2021-07-20 14:12:40,959 (lifecycleSupervisor-1-0) [INFO - org.apache.flume.source.NetcatSource.start(NetcatSource.java:166)] Created serverSocket:sun.nio.ch.ServerSocketChannelImpl[/127.0.0.1:8888]
2021-07-20 14:12:52,674 (SinkRunner-PollingRunner-DefaultSinkProcessor) [INFO - org.apache.flume.sink.LoggerSink.process(LoggerSink.java:95)] Event: { headers:{} body: 68 65 6C 6C 6F                                  hello }
2021-07-20 14:12:56,699 (SinkRunner-PollingRunner-DefaultSinkProcessor) [INFO - org.apache.flume.sink.LoggerSink.process(LoggerSink.java:95)] Event: { headers:{} body: 6E 69 68 61 6F                                  nihao }

#侦听目录中的新增文件
[root@yfm19 apache-flume-1.9.0-bin]# cat conf/flume-file.properties 
#####################################################################
## 监听目录中的新增文件
## this agent is consists of source which is r1 , sinks which is k1,
## channel which is c1
## 
## 这里面的a1 是flume一个实例agent的名字
#####################################################################

a1.sources = r1
a1.channels = c1
a1.sinks = k1

# 监听数据源的方式，这里采用监听目录中的新增文件

a1.sources.r1.type = spooldir
a1.sources.r1.spoolDir = /data/flume/test
a1.sources.r1.fileSuffix = .ok
# a1.sources.r1.deletePolicy = immediate
a1.sources.r1.deletePolicy = never
a1.sources.r1.fileHeader = true

# 采集的数据的下沉(落地)方式 通过日志
a1.sinks.k1.type = logger

# 描述channel的部分，使用内存做数据的临时存储
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100

# 使用channel将source和sink连接起来
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1

bin/flume-ng agent -c conf -n a1 -f conf/flume-file.properties -Dflume.root.logger=INFO,console

在配置文件的本地目录中增加文件，输入
hello wuyifan

此时可以查看flume agent启动终端的输出
2021-07-20 14:20:28,673 (pool-3-thread-1) [INFO - org.apache.flume.client.avro.ReliableSpoolingFileEventReader.rollCurrentFile(ReliableSpoolingFileEventReader.java:497)] Preparing to move file /data/flume/test/bb.txt to /data/flume/test/bb.txt.ok
2021-07-20 14:20:31,894 (SinkRunner-PollingRunner-DefaultSinkProcessor) [INFO - org.apache.flume.sink.LoggerSink.process(LoggerSink.java:95)] Event: { headers:{file=/data/flume/test/bb.txt} body: 68 65 6C 6C 6F 20 77 75 79 69 66 61 6E          hello wuyifan }