Flume安装配置及命令行操作

Flume下载、安装、配置

下载

   地址：http://flume.apache.org/download.html

安装

1. 查看当前服务器是否有flume组件

ps -ef|grep flume

2. 利用工具将压缩包上传至linux服务器相应路径
3. 解压

tar -zxvf apache-flume-1.9.0-bin.tar.gz

设置配置文件

# cd apache-flume-1.9.0-bin/conf
# touch file2kafka.conf
# vi file2kafka.conf

# define
a1.sources = r1
a1.sinks = k1
a1.channels = c1

# source
a1.sources.r1.type = exec
a1.sources.r1.command = tail -F -c +0 /usr/local/data/call.log
a1.sources.r1.shell = /bin/bash -c

# sink
a1.sinks.k1.type = org.apache.flume.sink.kafka.KafkaSink
a1.sinks.k1.kafka.bootstrap.servers = 172.16.23.200:9092,172.16.23.202:9092,172.16.23.203:9092
a1.sinks.k1.kafka.topic = test
a1.sinks.k1.kafka.flumeBatchSize = 20
a1.sinks.k1.kafka.producer.acks = 1
a1.sinks.k1.kafka.producer.linger.ms = 1

# channel
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100

# bind
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1

命令行操作

启动flume

# cd apache-flume-1.9.0-bin/
bin/flume-ng agent --conf conf --conf-file conf/file2kafka.conf --name a1 -Dflume.root.logger=INFO,console

通过flume将数据通传输到kafka

# cd /opt/kafka/bin
./kafka-console-producer.sh --broker-list hadoop102:9092 --topic test
# cd /usr/local
mkdir data
# cd data
touch call.log
echo 111 >> call.log
echo 222 >> call.log