Flume+Kafka日志传输

Flume+Kafka日志传输

一、flume配置

配置文件在conf/flume-conf.properties.template

a1.sources = s1
a1.channels = c1
a1.sink = k1

a1.sources.s1.type = exec
a1.sources.s1.command = tail -F /usr/tmp/logs/test.log
a1.sources.s1.channels = c1

#a1.channels.c1.type = memory
#a1.channels.c1.capacity = 1000
#a1.channels.c1.transactionCapacity = 100
a1.channels.c1.type = org.apache.flume.channel.kafka.KafkaChannel
a1.channels.c1.kafka.bootstrap.servers = 192.168.133.128:9092,192.168.133.128:9093,192.168.133.128:9094
a1.channels.c1.kafka.topic = test
a1.channels.c1.kafka.consumer.group.id = flume-consumer

a1.sinks.k1.channel = c1
a1.sinks.k1.type = org.apache.flume.sink.kafka.KafkaSink
a1.sinks.k1.kafka.topic = test
a1.sinks.k1.kafka.bootstrap.servers = 192.168.133.128:9092
a1.sinks.k1.kafka.flumeBatchSize = 20
a1.sinks.k1.kafka.producer.acks = 1
a