tcp实时传输kafka数据_flume通过tcp/udp采集数据并存到kafka配置及操作方式

运维开发网 https://www.qedev.com

2020-03-20 21:04

出处：网络作者：运维开发网整理

/*官方提供的kafka sink*/ a1.sinks.k1.channel = c1 a1.sinks.k1.type = org.apache.flume.sink.kafka.KafkaSink a1.sinks.k1.kafka.topic = mytopic a1.sinks.k1.kafka.bootstrap.servers = localhost:9092 a1.sinks.k1

/*官方提供的kafka sink*/ a1.sinks.k1.channel = c1 a1.sinks.k1.type = org.apache.flume.sink.kafka.KafkaSink a1.sinks.k1.kafka.topic = mytopic a1.sinks.k1.kafka.bootstrap.servers = localhost:9092 a1.sinks.k1.kafka.flumeBatchSize = 20 a1.sinks.k1.kafka.producer.acks = 1 a1.sinks.k1.kafka.producer.linger.ms = 1 a1.sinks.ki.kafka.producer.compression.type = snappy /**在kafka中创建一个名为 flumetest的topic*/ bin/kafka-console-consumer.sh --zookeeper  mpc5:2181,mpc6:2181,mpc7  --from-beginning --topic flumetest /**kafka中在控制台上消费消息队列的启动方式*/ bin/kafka-console-consumer.sh --zookeeper  mpc5:2181,mpc6:2181,mpc7  --from-beginning --topic flumetest /*比较好用的kafka sink 但是不可靠*/ producer.sinks.r.type = org.apache.flume.plugins.KafkaSink producer.sinks.r.metadata.broker.list=dn1:9092,dn2:9092,dn3:9092 producer.sinks.r.partition.key=0 producer.sinks.r.partitioner.class=org.apache.flume.plugins.SinglePartition producer.sinks.r.serializer.class=kafka.serializer.StringEncoder producer.sinks.r.request.required.acks=0 producer.sinks.r.max.message.size=1000000 producer.sinks.r.producer.type=sync producer.sinks.r.custom.encoding=UTF-8 producer.sinks.r.custom.topic.name=test /*一个配置文件开始*/ a1.sources = r1 a1.channels = c1 a1.sinks =k1 #source a1.sources.r1.type = netcat a1.sources.r1.bind = mpc1 a1.sources.r1.port = 11203 a1.sources.r1.channels = c1 # Use a channel which buffers events in memory a1.channels.c1.type = memory a1.channels.c1.capacity = 1000 a1.channels.c1.transactionCapacity = 100 #sink a1.sinks.k1.channel = c1 a1.sinks.k1.type = org.apache.flume.sink.kafka.KafkaSink a1.sinks.k1.kafka.topic = flumetest a1.sinks.k1.kafka.bootstrap.servers = mpc5:9092,mpc6:9092,mpc7:9092 a1.sinks.k1.kafka.flumeBatchSize = 20 a1.sinks.k1.kafka.producer.acks = 0 a1.sinks.k1.kafka.producer.compression.type = snappy a1.sinks.k1.kafka.producer.zk.connect=mpc5:2181,mpc6:2181,mpc7:2181 a1.sinks.k1.kafka.producer.serializer.class=kafka.serializer.StringEncoder /*一个配置文件结束*/ \x0D /*tcp采集源*/ a1.sources = r1 a1.channels = c1 a1.sources.r1.type = syslogtcp a1.sources.r1.port = 5140 a1.sources.r1.host = localhost a1.sources.r1.channels = c1 /*多个采集源的flume，分别是tcp和udp；sink到kafka中 开始*/ a1.sources = r1 r2 a1.channels = c1 a1.sinks =k1 #source a1.sources.r1.type = multiport_syslogtcp a1.sources.r1.channels = c1 a1.sources.r1.host = mpc1 a1.sources.r1.ports = 10001 10002 10003 a1.sources.r1.portHeader = port a1.sources.r2.type = syslogudp a1.sources.r2.port = 5140 a1.sources.r2.host = mpc1 a1.sources.r2.channels = c1 # Use a channel which buffers events in memory a1.channels.c1.type = memory a1.channels.c1.capacity = 1000 a1.channels.c1.transactionCapacity = 100 #sink a1.sinks.k1.channel = c1 a1.sinks.k1.type = org.apache.flume.sink.kafka.KafkaSink a1.sinks.k1.kafka.topic = flumetest a1.sinks.k1.kafka.bootstrap.servers = mpc5:9092,mpc6:9092,mpc7:9092 a1.sinks.k1.kafka.flumeBatchSize = 20 a1.sinks.k1.kafka.producer.acks = 0 a1.sinks.k1.kafka.producer.compression.type = snappy a1.sinks.k1.kafka.producer.zk.connect=mpc5:2181,mpc6:2181,mpc7:2181 a1.sinks.k1.kafka.producer.serializer.class=kafka.serializer.StringEncoder /*多个采集源的flume，分别是tcp和udp；sink到kafka中 结束*/ /*启动flume的方法*/ bin/flume-ng agent -n a1 -c conf -f conf/flume-conf.properties /*删除一个topic在zookeeper上的元数据*/ bin/kafka-run-class.sh kafka.admin.DeleteTopicCommand --topic mytri1 --zookeeper mpc5:2181 /**flume到kafka利用id来进行分区的配置*/ a1.sources = r1 r2 a1.channels = c1 a1.sinks =k1 #source a1.sources.r1.type = multiport_syslogtcp a1.sources.r1.channels = c1 a1.sources.r1.host = mpc1 a1.sources.r1.ports = 10001 10002 10003 a1.sources.r1.portHeader = port a1.sources.r1.interceptors = i2 a1.sources.r1.interceptors.i2.type=org.apache.flume.sink.solr.morphline.UUIDInterceptor$Builder a1.sources.r1.interceptors.i2.headerName=key a1.sources.r1.interceptors.i2.preserveExisting=false a1.sources.r2.type = syslogudp a1.sources.r2.port = 5140 a1.sources.r2.host = mpc1 a1.sources.r2.channels = c1 a1.sources.r2.interceptors = i2 a1.sources.r2.interceptors.i2.type=org.apache.flume.sink.solr.morphline.UUIDInterceptor$Builder a1.sources.r2.interceptors.i2.headerName=key a1.sources.r2.interceptors.i2.preserveExisting=false # Use a channel which buffers events in memory a1.channels.c1.type = memory a1.channels.c1.capacity = 1000 a1.channels.c1.transactionCapacity = 100 #sink a1.sinks.k1.channel = c1 a1.sinks.k1.type = org.apache.flume.sink.kafka.KafkaSink a1.sinks.k1.kafka.topic = mytri3 a1.sinks.k1.kafka.bootstrap.servers = mpc5:9092,mpc6:9092,mpc7:9092 a1.sinks.k1.kafka.flumeBatchSize = 20 a1.sinks.k1.kafka.producer.acks = 0 a1.sinks.k1.kafka.producer.zk.connect=mpc5:2181,mpc6:2181,mpc7:2181 a1.sinks.k1.kafka.producer.serializer.class=kafka.serializer.StringEncoder

分享到

0

0

0