服务器上部署flume
1、PC2/PC3上,/data/下创建flume文件夹,将包解压
2、复制${FLUME_HOME}/conf/flume-env.sh.template 为 flume-env.sh
3、修改 ${FLUME_HOME}/conf/flume-env.sh 文件
export JAVA_HOME=/data/jdk/jdk1.8.0_40
export JAVA_OPTS="-Xms256m -Xmx1024m -Xss256k -Xmn128m -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-UseGCOverheadLimit"
FLUME_CLASSPATH="/data/flume/apache-flume-1.7.0-bin/conf"
检查是否配置成功
4、PC3:${FLUME_HOME}/conf下,创建 avro-a3002.properties 文件
a3002.sources = s1
a3002.sinks = k1
a3002.channels = c1
a3002.sources.s1.type = avro
a3002.sources.s1.bind = 192.168.47.133
a3002.sources.s1.port = 3002
a3002.sources.s1.deserializer.outputCharset = UTF-8
a3002.sources.s1.inputCharset = UTF-8
a3002.sinks.k1.type = avro
a3002.sinks.k1.hostname = 192.168.47.133
a3002.sinks.k1.port = 3003
a3002.sinks.k1.batch-size=2000
a3002.channels.c1.type = memory
a3002.channels.c1.capacity = 40000
a3002.channels.c1.transactionCapacity = 2000
a3002.sources.s1.channels = c1
a3002.sinks.k1.channel = c1
PC2:${FLUME_HOME}/conf下,创建 avro-a3001.properties 文件
a3001.sources = s1
a3001.sinks = k1
a3001.channels = c1
a3001.sources.s1.type = avro
a3001.sources.s1.bind = 192.168.47.132
a3001.sources.s1.port = 3001
a3001.sources.s1.deserializer.outputCharset = UTF-8
a3001.sources.s1.inputCharset = UTF-8
a3001.sinks.k1.type = avro
a3001.sinks.k1.hostname = 192.168.47.132
a3001.sinks.k1.port = 3003
a3001.sinks.k1.batch-size=2000
a3001.channels.c1.type = memory
a3001.channels.c1.capacity = 40000
a3001.channels.c1.transactionCapacity = 2000
a3001.sources.s1.channels = c1
a3001.sinks.k1.channel = c1
5、 PC3:${FLUME_HOME}/conf下, 创建 avro-hdfs.properties 文件
a3003.sources = s1
a3003.sinks = k1
a3003.channels = c1
# set channel for sources, sinks
# properties of s1
a3003.sources.s1.type = avro
a3003.sources.s1.bind = 192.168.47.133
a3003.sources.s1.port = 3003
a3003.sources.s1.deserializer.outputCharset = UTF-8
a3003.sources.s1.inputCharset = UTF-8
# prorperties of k1
a3003.sinks.k1.type = hdfs
a3003.sinks.k1.hdfs.path = /flume/logs/%Y-%m-%d/%H
a3003.sinks.k1.hdfs.filePrefix = log%H
a3003.sinks.k1.hdfs.useLocalTimeStamp = true
a3003.sinks.k1.hdfs.writeFormat = Text
a3003.sinks.k1.hdfs.fileType = DataStream
a3003.sinks.k1.hdfs.rollCount = 0
a3003.sinks.k1.hdfs.rollSize = 134217728
a3003.sinks.k1.hdfs.rollInterval = 600
a3003.sinks.k1.hdfs.batchSize = 10000
a3003.sinks.k1.hdfs.threadsPoolSize = 10
a3003.sinks.k1.hdfs.idleTimeount = 0
a3003.sinks.k1.hdfs.minBlockReplicas = 1
# properties of c1
a3003.channels.c1.type = file
a3003.channels.c1.maxFileSize = 67108864
a3003.channels.c1.transactionCapacity = 10000
a3003.channels.c1.capacity = 50000
a3003.channels.c1.dataDirs = /data/flume/data/channelsData/data
a3003.channels.c1.checkpointDir = /data/flume/data/channelsData/checkpoint
# properties of s1,k1's channel
a3003.sources.s1.channels = c1
a3003.sinks.k1.channel = c1
PC2:${FLUME_HOME}/conf下,创建 avro-hdfs.properties 文件
a3001.sources = s1
a3001.sinks = k1
a3001.channels = c1
# set channel for sources, sinks
# properties of s1
a3001.sources.s1.type = avro
a3001.sources.s1.bind = 192.168.47.132
a3001.sources.s1.port = 3003
a3001.sources.s1.deserializer.outputCharset = UTF-8
a3001.sources.s1.inputCharset = UTF-8
# prorperties of k1
a3001.sinks.k1.type = hdfs
a3001.sinks.k1.hdfs.path = /flume/logs/%Y-%m-%d/%H
a3001.sinks.k1.hdfs.filePrefix = log%H
a3001.sinks.k1.hdfs.useLocalTimeStamp = true
a3001.sinks.k1.hdfs.writeFormat = Text
a3001.sinks.k1.hdfs.fileType = DataStream
a3001.sinks.k1.hdfs.rollCount = 0
a3001.sinks.k1.hdfs.rollSize = 134217728
a3001.sinks.k1.hdfs.rollInterval = 600
a3001.sinks.k1.hdfs.batchSize = 10000
a3001.sinks.k1.hdfs.threadsPoolSize = 10
a3001.sinks.k1.hdfs.idleTimeount = 0
a3001.sinks.k1.hdfs.minBlockReplicas = 1
# properties of c1
a3001.channels.c1.type = file
a3001.channels.c1.maxFileSize = 67108864
a3001.channels.c1.transactionCapacity = 10000
a3001.channels.c1.capacity = 50000
a3001.channels.c1.dataDirs = /data/flume/data/channelsData/data
a3001.channels.c1.checkpointDir = /data/flume/data/channelsData/checkpoint
# properties of s1,k1's channel
a3001.sources.s1.channels = c1
a3001.sinks.k1.channel = c1
新增两个文件
6、启动flume
使用nohup启动挂起flume:(PC3/PC2分别启动,注意修改配置文件名)
PC3启动:
nohup bin/flume-ng agent --conf conf --conf-file conf/avro-hdfs.properties --name a3002 -Dflume.root.logger=DEBUG,console -Dorg.apache.flume.log.printconfig=true -Dorg.apache.flume.log.rawdata=true > hdfs.out &
nohup bin/flume-ng agent --conf conf --conf-file conf/avro-a3002.properties --name a3002 -Dflume.root.logger=DEBUG,console -Dorg.apache.flume.log.printconfig=true -Dorg.apache.flume.log.rawdata=true > avro.out &
PC2启动:
nohup bin/flume-ng agent --conf conf --conf-file conf/avro-hdfs.properties --name a3001 -Dflume.root.logger=DEBUG,console -Dorg.apache.flume.log.printconfig=true -Dorg.apache.flume.log.rawdata=true > hdfs.out &
nohup bin/flume-ng agent --conf conf --conf-file conf/avro-a3001.properties --name a3001 -Dflume.root.logger=DEBUG,console -Dorg.apache.flume.log.printconfig=true -Dorg.apache.flume.log.rawdata=true > avro.out &
启动,jps查询,有一个Application的进程