CentOs7 flume+kafka

tar -zxvf apache-flume-1.7.0-bin.tar.gz
mv apache-flume-1.7.0-bin flume
cd /flume/conf/
mv flume-env.sh.template flume-env.sh
vi flume-env.sh
export JAVA_HOME=/usr/java/jdk1.8.0_261
./flume-ng version
Flume 1.7.0
Source code repository: https://git-wip-us.apache.org/repos/asf/flume.git
Revision: 511d868555dd4d16e6ce4fedc72c2d1454546707
Compiled by bessbd on Wed Oct 12 20:51:10 CEST 2016
From source with checksum 0d21b3ffdc55a07e1d08875872c00523

单层flume，使用taildir source从日志文件中采集日志，使用kafka sink传输给kafka
配置文件根据flume官方文档

# agent的组件名称
a1.sources = r1 
a1.channels = c1 
a1.sinks = k1

# source配置
a1.sources.r1.type = TAILDIR
a1.sources.r1.channels = c1
a1.sources.r1.positionFile = /opt/flume/checkpoint/behavior/taildir_position.json
a1.sources.r1.filegroups = f1 f2
a1.sources.r1.filegroups.f1 = /opt/apache-tomcat-7.0.72-01/logs/LogsCollect/atguigu.log
a1.sources.r1.filegroups.f2 = /opt/apache-tomcat-7.0.72-02/logs/LogsCollect/atguigu.log
a1.sources.r1.fileHeader = true

# channel配置
a1.channels.c1.type = file
a1.channels.c1.checkpointDir = /opt/flume/checkpoint/behavior_collect 
a1.channels.c1.dataDirs = /opt/flume/data/behavior_collect 
a1.channels.c1.maxFileSize = 104857600
a1.channels.c1.capacity = 90000000
a1.channels.c1.keep-alive = 60

# sink配置
a1.sinks.k1.type = org.apache.flume.sink.kafka.KafkaSink
a1.sinks.k1.topic = analysis-test 
a1.sinks.k1.kafka.bootstrap.servers = 192.168.44.10:9092,192.168.44.11:9092,192.168.44.12:9092
a1.sinks.k1.requiredAcks = 1 
a1.sinks.k1.kafka.producer.type = sync 
a1.sinks.k1.batchSize = 1
a1.sinks.k1.channel = c1

测试
启动

cd /opt/apache-tomcat-7.0.72-01/bin/
./start.sh
cd /opt/apache-tomcat-7.0.72-02/bin/
./start.sh
cd /usr/local/webserver/nginx/sbin/
./nginx
cd /opt/flume/
./bin/flume-ng agent -n a1 -c conf/ -f conf/flume-analysis.conf
cd /usr/local/kafka/bin/
./bin/zookeeper-server-start.sh ./config/zookeeper.properties
./bin/kafka-server-start.sh -daemon ./config/server.properties
./bin/kafka-topics.sh --zookeeper node2:2181 --list
./bin/kafka-topics.sh --zookeeper node2:2181 --create --replication-factor 1 --partitions 1 --topic analysis-test
./bin/kafka-console-consumer.sh --zookeeper node2:2181 --topic analysis-test

关闭

停止程序
停止kafka
停止zookeeper
停止flume
停止nginx
停止tomcat