flume日志写入到hdfs按时间分割

exec-hdfs-agent.conf:

exec-hdfs-agent.sources = exec-source
exec-hdfs-agent.sinks = hdfs-sink
exec-hdfs-agent.channels = memory-channel

# Describe/configure the source
exec-hdfs-agent.sources.exec-source.type = exec
exec-hdfs-agent.sources.exec-source.command = tail -F ~/logs/access.log
exec-hdfs-agent.sources.exec-source.shell = /bin/bash -c

# Describe the sink
exec-hdfs-agent.sinks.hdfs-sink.type = hdfs
exec-hdfs-agent.sinks.hdfs-sink.hdfs.path = hdfs://hadoop000:8020/data/flume/events/%y-%m-%d/%H%M/%S
exec-hdfs-agent.sinks.hdfs-sink.hdfs.fileType=DataStream
exec-hdfs-agent.sinks.hdfs-sink.hdfs.writeFormat=Text
exec-hdfs-agent.sinks.hdfs-sink.hdfs.batchSize=100
exec-hdfs-agent.sinks.hdfs-sink.hdfs.filePrefix = events-
exec-hdfs-agent.sinks.hdfs-sink.hdfs.round = true
exec-hdfs-agent.sinks.hdfs-sink.hdfs.roundValue = 1
exec-hdfs-agent.sinks.hdfs-sink.hdfs.roundUnit = minute
exec-hdfs-agent.sinks.hdfs-sink.hdfs.useLocalTimeStamp = true

# Use a channel which buffers events in memory
exec-hdfs-agent.channels.memory-channel.type = memory

# Bind the source and sink to the channel
exec-hdfs-agent.sources.exec-source.channels = memory-channel
exec-hdfs-agent.sinks.hdfs-sink.channel = memory-channel

命令解释：
文件夹按/%y-%m-%d/%H%M/%S进行分割。
按一分钟进行四舍五入。

运行结果：

[hadoop@hadoop000 logs]$ hadoop fs -ls /data/flume/events/21-02-08
Found 4 items
drwxr-xr-x   - hadoop supergroup          0 2021-02-08 14:22 /data/flume/events/21-02-08/1422
drwxr-xr-x   - hadoop supergroup          0 2021-02-08 14:23 /data/flume/events/21-02-08/1423
drwxr-xr-x   - hadoop supergroup          0 2021-02-08 14:24 /data/flume/events/21-02-08/1424
drwxr-xr-x   - hadoop supergroup          0 2021-02-08 14:25 /data/flume/events/21-02-08/1425

[hadoop@hadoop000 logs]$ hadoop fs -ls /data/flume/events/21-02-08/1425/00
Found 7 items
-rw-r--r--   1 hadoop supergroup        300 2021-02-08 14:25 /data/flume/events/21-02-08/1425/00/events-.1612765500050
-rw-r--r--   1 hadoop supergroup        300 2021-02-08 14:25 /data/flume/events/21-02-08/1425/00/events-.1612765500051
-rw-r--r--   1 hadoop supergroup        300 2021-02-08 14:25 /data/flume/events/21-02-08/1425/00/events-.1612765500052
-rw-r--r--   1 hadoop supergroup        300 2021-02-08 14:25 /data/flume/events/21-02-08/1425/00/events-.1612765500053
-rw-r--r--   1 hadoop supergroup        300 2021-02-08 14:25 /data/flume/events/21-02-08/1425/00/events-.1612765500054
-rw-r--r--   1 hadoop supergroup        300 2021-02-08 14:25 /data/flume/events/21-02-08/1425/00/events-.1612765500055
-rw-r--r--   1 hadoop supergroup        270 2021-02-08 14:25 /data/flume/events/21-02-08/1425/00/events-.1612765500056