# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# The configuration file needs to define the sources,
# the channels and the sinks.
# Sources, channels and sinks are defined per agent,
# in this case called 'agent'
# example.conf: A single-node Flume configuration
# Name the components on this agent
#把agent起个名叫a2,sources叫r2,sinks叫k2.hdfs,channels叫c2
a2.sources = r2
a2.sinks = k2
a2.channels = c2
#监听数据来源为本地的4444端口
# Describe/configure the source
a2.sources.r2.type = exec
a2.sources.r2.command = tail -f /opt/modules/hive-0.13.1-cdh5.3.6/logs/hive.log
#/bin/bash -c echo `date` 可以写成 echo /bin/bash -c date 就是它后面跟一个可执行脚本,来执行上面的监控日志的命令
a2.sources.r2.shell = /bin/bash -c
#logger是流传输过来解码后的数据
# Describe the sink
a2.sinks.k2.type = hdfs
a2.sinks.k2.hdfs.path = hdfs://192.168.1.20:8020/flume/%Y%m%d/%H
#上传文件的前缀
a2.sinks.k2.hdfs.filePrefix = events-hive-
#是否按照时间滚动文件夹
a2.sinks.k2.hdfs.round = true
#多长时间创建一个新的文件夹 1小时
a2.sinks.k2.hdfs.roundVlaue = 1
#重新定义时间单位
a2.sinks.k2.hdfs.roundUnit = hour
#是否使用本地时间戳
a2.sinks.k2.hdfs.useLocalTimeStamp = true
#积攒多个event的时候才flush到hdfs上
a2.sinks.k2.hdfs.batchSize = 1000
#设置文件类型,可支持压缩
a2.sinks.k2.hdfs.fileType = DataStream
#多久生成一个新文件 10分钟
a2.sinks.k2.hdfs.rollInterval = 600
#设置文件块多大.即使没到600秒,但是达到这个大小了也生成新文件
a2.sinks.k2.hdfs.rollSize = 134217700
#文件的滚动与event数量无关
a2.sinks.k2.hdfs.rollCount = 0
#最小冗余数
a2.sinks.k2.hdfs.minBlockReplicas = 1
#channels阶段以内存的形式保存数据 event数量100
# Use a channel which buffers events in memory
a2.channels.c2.type = memory
a2.channels.c2.capacity = 1000
a2.channels.c2.transactionCapacity = 100
#把source和sink和channel对接 source可以对接多个channels sinks只能对接一个channel
# Bind the source and sink to the channel
a2.sources.r2.channels = c2
a2.sinks.k2.channel = c2