flume启动配置例子

flume.conf

​
agent.sources=kafka_source_1
agent.channels=memory_channel_1
agent.sinks=kafka_sink_1

agent.sources.kafka_source_1.kafka.bootstrap.servers=bo-hadoop003.bonc.com:9092,bo-hadoop004.bonc.com:9092,bo-hadoop005.bonc.com:9092,bo-hadoop006.bonc.com:9092,bo-hadoop007.bonc.com:9092,bo-hadoop008.bonc.com:9092,bo-hadoop009.bonc.com:9092
agent.sources.kafka_source_1.channels=memory_channel_1
agent.sources.kafka_source_1.interceptors=interceptorn84762951
agent.sources.kafka_source_1.kafka.consumer.security.protocol=SASL_PLAINTEXT
agent.sources.kafka_source_1.setTopicHeader=true
agent.sources.kafka_source_1.type=org.apache.flume.source.kafka.KafkaSource
agent.sources.kafka_source_1.maxBackoffSleep=1000
agent.sources.kafka_source_1.kafka.consumer.sasl.kerberos.service.name=kafka
agent.sources.kafka_source_1.useFlumeEventFormat=false
agent.sources.kafka_source_1.interceptors.interceptorn84762951.type=com.chinaunicom.roam.flume.interceptor.RoamDataOfInterceptorToZTEC$Builder
agent.sources.kafka_source_1.topicHeader=O_epc_event_roam
agent.sources.kafka_source_1.name=Source
agent.sources.kafka_source_1.interceptors.interceptorn84762951.customParam=[]
agent.sources.kafka_source_1.interceptors.interceptorn84762951.interceptorName=i1
agent.sources.kafka_source_1.batchDurationMillis=3000
agent.sources.kafka_source_1.interceptors.interceptorn84762951.uploadJarFile=false
agent.sources.kafka_source_1.kafka.consumer.sasl.mechanism=GSSAPI
agent.sources.kafka_source_1.kafka.topics=O_epc_event_roam
agent.sources.kafka_source_1.backoffSleepIncrement=1000
agent.sources.kafka_source_1.batchSize=5000
agent.sources.kafka_source_1.kafka.consumer.group.id=O_epc_event_roam-20190903

agent.channels.memory_channel_1.type=Memory
agent.channels.memory_channel_1.name=Channel
agent.channels.memory_channel_1.capacity=5000000
agent.channels.memory_channel_1.byteCapacityBufferPercentage=20
agent.channels.memory_channel_1.keep-alive=3
agent.channels.memory_channel_1.transactionCapacity=3000000
agent.channels.memory_channel_1.byteCapacity=1048576000

agent.sinks.kafka_sink_1.kafka.bootstrap.servers=bo-hadoop003.bonc.com:9092,bo-hadoop004.bonc.com:9092,bo-hadoop005.bonc.com:9092,bo-hadoop006.bonc.com:9092,bo-hadoop007.bonc.com:9092,bo-hadoop008.bonc.com:9092,bo-hadoop009.bonc.com:9092
agent.sinks.kafka_sink_1.type=org.apache.flume.sink.kafka.KafkaSink
agent.sinks.kafka_sink_1.kafka.producer.security.protocol=SASL_PLAINTEXT
agent.sinks=kafka_sink_1
agent.sinks.kafka_sink_1.name=Sink
agent.sinks.kafka_sink_1.allowTopicOverride=false
agent.sinks.kafka_sink_1.kafka.producer.sasl.mechanism=GSSAPI
agent.sinks.kafka_sink_1.kafka.topic=O_ROAM_234G_MERGE
agent.sinks.kafka_sink_1.kafka.producer.sasl.kerberos.service.name=kafka
agent.sinks.kafka_sink_1.useFlumeEventFormat=false
agent.sinks.kafka_sink_1.defaultPartitionId=
agent.sinks.kafka_sink_1.topicHeader=
agent.sinks.kafka_sink_1.kafka.producer.acks=1
agent.sinks.kafka_sink_1.channel=memory_channel_1
agent.sinks.kafka_sink_1.flumeBatchSize=5000

 

flume-env.sh

export FLUMELOGNAME=/home/behb/beh-agent-shanghai/bdc-agent-shanghai/tasks/06c098df6e72ead4016e7da29245000b/logs/task.log
export FLUME_HOME=""
export JAVA_OPTS="${JAVA_OPTS} -Djava.security.krb5.conf=/etc/krb5.conf"
export JAVA_OPTS="${JAVA_OPTS} -Djava.security.auth.login.config=/home/behb/beh-agent/bdc-agent/kerberosfile/kafka_client_jaas_210.conf"
export JAVA_OPTS="${JAVA_OPTS} -Xms12G -Xmx12G -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false"
FLUME_CLASSPATH="/home/behb/beh-agent-shanghai/bdc-agent-shanghai/tasks/06c098df6e72ead4016e7da29245000b/libs/*:/home/behb/beh-agent-shanghai/bdc-agent-shanghai/flume/lib/*:$FLUME_CLASSPATH"

log4j.properties

flume.root.logger=INFO,LOGFILE

log4j.logger.org.apache.flume.lifecycle = INFO
log4j.logger.org.jboss = WARN
log4j.logger.org.mortbay = INFO
log4j.logger.org.apache.avro.ipc.NettyTransceiver = WARN
log4j.logger.org.apache.hadoop = INFO
log4j.logger.org.apache.hadoop.hive = ERROR

log4j.rootLogger=${flume.root.logger}


log4j.appender.LOGFILE=org.apache.log4j.RollingFileAppender
log4j.appender.LOGFILE.MaxFileSize=3MB
log4j.appender.LOGFILE.MaxBackupIndex=60
log4j.appender.LOGFILE.File=${FLUMELOGNAME}
log4j.appender.LOGFILE.layout=org.apache.log4j.PatternLayout
log4j.appender.LOGFILE.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} %-5p [%t] (%C.%M:%L) %x - %m%n


log4j.appender.DAILY=org.apache.log4j.rolling.RollingFileAppender
log4j.appender.DAILY.rollingPolicy=org.apache.log4j.rolling.TimeBasedRollingPolicy
log4j.appender.DAILY.rollingPolicy.ActiveFileName=${FLUMELOGNAME}
log4j.appender.DAILY.rollingPolicy.FileNamePattern=${FLUMELOGNAME}.%d{yyyy-MM-dd}
log4j.appender.DAILY.layout=org.apache.log4j.PatternLayout
log4j.appender.DAILY.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} %-5p [%t] (%C.%M:%L) %x - %m%n


log4j.appender.console=org.apache.log4j.ConsoleAppender
log4j.appender.console.target=System.err
log4j.appender.console.layout=org.apache.log4j.PatternLayout
log4j.appender.console.layout.ConversionPattern=%d (%t) [%p - %l] %m%n

启动命令：

在flume/conf/flume-env.sh添加krb5和jaas路径
export JAVA_OPTS="${JAVA_OPTS} -Djava.security.krb5.conf=/opt/beh/core/flume/flume/conf/krb5.conf"
export JAVA_OPTS="${JAVA_OPTS} -Djava.security.auth.login.config=/opt/beh/core/flume/flume/conf/kafka_client_jaas.conf"


flume启动命令
nohup /home/flume/flume-1.7/bin/flume-ng agent  \
--conf /home/flume/flume-1.7/conf  \
--conf-file /home/flume/flume-1.7/key/flume-exec.conf     \
--name agent-streaming \
-Dflume.root.logger=INFO,console >/home/flume/flume-1.7/logs/log &

 

flume-kafka.conf

agent.sources=r1
agent.channels=c1
agent.sinks=k1
    
agent.sources.r1.type=spooldir
agent.sources.r1.spoolDir = /pkgcbss1/shcbssftp_actual/data_merge/flume2bo_seashield
#agent.sources.r1.spoolDir = /pkgcbss1/shcbssftp_actual/data_merge/test_seashield
agent.sources.r1.channels = c1
agent.sources.r1.batchSize = 10000
agent.sources.r1.deserializer.maxLineLength = 204800
agent.sources.r1.inputCharset = latin1
#agent.sources.r1.deletePolicy=immediate  latin1

agent.channels.c1.type = memory
agent.channels.c1.capacity = 500000
agent.channels.c1.transactionCapacity = 10000
#agent.channels.c1.byteCapacityBufferPercentage=70
agent.channels.c1.byteCapacity = 104857600
    
agent.sinks.k1.channel=c1
agent.sinks.k1.type=org.apache.flume.sink.kafka.KafkaSink
agent.sinks.k1.kafka.topic=B_SHIELD
agent.sinks.k1.kafka.bootstrap.servers=bo-hadoop003.bonc.com:9092,bo-hadoop004.bonc.com:9092,bo-hadoop005.bonc.com:9092,bo-hadoop006.bonc.com:9092,bo-hadoop007.bonc.com:9092,bo-hadoop008.bonc.com:9092,bo-hadoop009.bonc.com:9092
agent.sinks.k1.kafka.flumeBatchSize = 10000
agent.sinks.k1.kafka.producer.acks = 1
agent.sinks.k1.kafka.producer.security.protocol = SASL_PLAINTEXT
agent.sinks.k1.kafka.producer.sasl.mechanism = GSSAPI
agent.sinks.k1.kafka.producer.sasl.kerberos.service.name = kafka

 

 

flume自定义组件放到lib目录下或者新建plugins.d目录下自动管理

1. lib 目录

将maven项目打成jar包，将jar 包放到flume的lib 目录下。

此方法简单粗暴，缺点就是jar 包不易管理

2. 官方给定目录

官方建议在flume的 plugins.d (plugins.d 目录需要自己创建)目录下创建 一个自己定义的目录，在自定义的目录下新建 lib 和 libext 文件夹，lib 文件夹为放自定义组件的jar包，libext 文件夹下放 自定义组件的依赖包。

1. flume-1.7.0/plugins.d/

2. flume-1.7.0/plugins.d/custom-MysqlSink/

3. flume-1.7.0/plugins.d/custom-MysqlSink/lib/mysql-sink.jar

4. flume-1.7.0/plugins.d/custom-MysqlSink/libext/mysql-connector-java-6.0.5.jar

• 1
• 2
• 3
• 4

此方法方便，易管理，就是需要自己建目录。

3. 自我管理

把jar包放在自己想放的目录，但是代价就是启动时需要通过 -C 指定jar 包位置。

在启动的时候 直接加载jar 包 ，依赖的jar 包可以用“，”号隔开

./flume-ng agent -c /opt/apps/flume-1.7.0/conf -f /opt/apps/flume-1.7.0/conf/mysqlSink.conf -n agent1 -C /opt/apps/flumeInterceptor.jar  -Dflume.root.logger=INFO,console 

• 1

此方法不太方便，需要在启动时指定位置，比较麻烦。