Flume日志收集详解

flume采集客户端flume-client-1.2.2.zip

通用Flume配置说明

1. 日志文件路径配置(正式环境由运维负责配置)
   ##日志文件路径
   client.sources.serverLogSource.file=D://datasync-server//work//logs//datasync.log
   ##position路径，一般指定为flume目录
   client.sources.serverLogSource.positionDir=D://flume//server-position
    
2. 日志内容属性配置(正式环境部署前由开发负责给定值)
   ##静态数据
   client.sources.serverLogSource.interceptors.i1.properties=account=D2_FS_28,app =2.25,system=datasync,module=fileSyncV2,server=D2_FS_28
   ##kafka的topic，生产环境需提前申请，按系统分
   client.sources.serverLogSource.interceptors.i1.topic= 
   ##es的index名，全小写,按系统分
   client.sources.serverLogSource.interceptors.i1.index=
    
3. 全量配置说明

   #公司内网同步服务端
   client.sources = serverLogSource
   client.channels = serverLogChannel
   client.sinks = serverLogSink

   client.sources.serverLogSource.channels = serverLogChannel
   client.sources.serverLogSource.type = com.gildata.flume.source.file.FileSource
   ##日志文件路径
   client.sources.serverLogSource.file=D://datasync-server//work//logs//datasync.log
   ##position路径，一般指定为flume目录
   client.sources.serverLogSource.positionDir=D://flume//server-position
   client.sources.serverLogSource.printTime=60

   client.sources.serverLogSource.interceptors = i1
   client.sources.serverLogSource.interceptors.i1.type = com.gildata.flume.interceptor.GildataClientInterceptorBuilder
   ##静态填充，k1=v1,k2=v2，都会写入到日志中心
   client.sources.serverLogSource.interceptors.i1.properties= account=D2_FS_28,app =2.25,system=datasync,module=fileSyncV2,server=D2_FS_28
   ##kafka的topic，生产环境需提前申请，按系统分
   client.sources.serverLogSource.interceptors.i1.topic = 
   ##es的index名，全小写,按系统分
   client.sources.serverLogSource.interceptors.i1.index=

   client.channels.serverLogChannel.type=memory
   client.channels.serverLogChannel.capacity=1000000
   client.channels.serverLogChannel.transactionCapacity=10000

   client.sinks.serverLogSink.channel = serverLogChannel
   client.sinks.serverLogSink.type = avro
   ##flume-server所在服务器 （运维填写）
   client.sinks.serverLogSink.hostname = 10.106.0.138
   ##flume-server端口 （运维填写）
   client.sinks.serverLogSink.port = 5151
   client.sinks.serverLogSink.compression-type=deflate
   client.sinks.serverLogSink.compression-level=6