Dubbo框架 - 4 - 服务器部署(3)

服务器上部署flume

1、PC2/PC3上，/data/下创建flume文件夹，将包解压

2、复制${FLUME_HOME}/conf/flume-env.sh.template 为 flume-env.sh

3、修改 ${FLUME_HOME}/conf/flume-env.sh 文件

export JAVA_HOME=/data/jdk/jdk1.8.0_40
export JAVA_OPTS="-Xms256m -Xmx1024m -Xss256k -Xmn128m -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-UseGCOverheadLimit"
FLUME_CLASSPATH="/data/flume/apache-flume-1.7.0-bin/conf"

检查是否配置成功

4、PC3:${FLUME_HOME}/conf下，创建 avro-a3002.properties 文件

a3002.sources = s1
a3002.sinks = k1
a3002.channels = c1

a3002.sources.s1.type = avro
a3002.sources.s1.bind = 192.168.47.133
a3002.sources.s1.port = 3002
a3002.sources.s1.deserializer.outputCharset = UTF-8
a3002.sources.s1.inputCharset = UTF-8

a3002.sinks.k1.type = avro
a3002.sinks.k1.hostname = 192.168.47.133
a3002.sinks.k1.port = 3003
a3002.sinks.k1.batch-size=2000

a3002.channels.c1.type = memory
a3002.channels.c1.capacity = 40000
a3002.channels.c1.transactionCapacity = 2000

a3002.sources.s1.channels = c1
a3002.sinks.k1.channel = c1

PC2:${FLUME_HOME}/conf下，创建 avro-a3001.properties 文件

a3001.sources = s1
a3001.sinks = k1
a3001.channels = c1

a3001.sources.s1.type = avro
a3001.sources.s1.bind = 192.168.47.132
a3001.sources.s1.port = 3001
a3001.sources.s1.deserializer.outputCharset = UTF-8
a3001.sources.s1.inputCharset = UTF-8

a3001.sinks.k1.type = avro
a3001.sinks.k1.hostname = 192.168.47.132
a3001.sinks.k1.port = 3003
a3001.sinks.k1.batch-size=2000

a3001.channels.c1.type = memory
a3001.channels.c1.capacity = 40000
a3001.channels.c1.transactionCapacity = 2000

a3001.sources.s1.channels = c1
a3001.sinks.k1.channel = c1

5、 PC3:${FLUME_HOME}/conf下， 创建 avro-hdfs.properties 文件

a3003.sources = s1
a3003.sinks = k1
a3003.channels = c1

# set channel for sources, sinks
# properties of s1
a3003.sources.s1.type = avro
a3003.sources.s1.bind = 192.168.47.133
a3003.sources.s1.port = 3003
a3003.sources.s1.deserializer.outputCharset = UTF-8
a3003.sources.s1.inputCharset = UTF-8

# prorperties of k1
a3003.sinks.k1.type = hdfs
a3003.sinks.k1.hdfs.path = /flume/logs/%Y-%m-%d/%H
a3003.sinks.k1.hdfs.filePrefix = log%H
a3003.sinks.k1.hdfs.useLocalTimeStamp = true
a3003.sinks.k1.hdfs.writeFormat = Text
a3003.sinks.k1.hdfs.fileType = DataStream
a3003.sinks.k1.hdfs.rollCount = 0
a3003.sinks.k1.hdfs.rollSize = 134217728
a3003.sinks.k1.hdfs.rollInterval = 600
a3003.sinks.k1.hdfs.batchSize = 10000
a3003.sinks.k1.hdfs.threadsPoolSize = 10
a3003.sinks.k1.hdfs.idleTimeount = 0
a3003.sinks.k1.hdfs.minBlockReplicas = 1

# properties of c1
a3003.channels.c1.type = file
a3003.channels.c1.maxFileSize = 67108864
a3003.channels.c1.transactionCapacity = 10000
a3003.channels.c1.capacity = 50000
a3003.channels.c1.dataDirs = /data/flume/data/channelsData/data
a3003.channels.c1.checkpointDir = /data/flume/data/channelsData/checkpoint

# properties of s1,k1's channel
a3003.sources.s1.channels = c1
a3003.sinks.k1.channel = c1

PC2:${FLUME_HOME}/conf下，创建 avro-hdfs.properties 文件

a3001.sources = s1
a3001.sinks = k1
a3001.channels = c1

# set channel for sources, sinks
# properties of s1
a3001.sources.s1.type = avro
a3001.sources.s1.bind = 192.168.47.132
a3001.sources.s1.port = 3003
a3001.sources.s1.deserializer.outputCharset = UTF-8
a3001.sources.s1.inputCharset = UTF-8

# prorperties of k1
a3001.sinks.k1.type = hdfs
a3001.sinks.k1.hdfs.path = /flume/logs/%Y-%m-%d/%H
a3001.sinks.k1.hdfs.filePrefix = log%H
a3001.sinks.k1.hdfs.useLocalTimeStamp = true
a3001.sinks.k1.hdfs.writeFormat = Text
a3001.sinks.k1.hdfs.fileType = DataStream
a3001.sinks.k1.hdfs.rollCount = 0
a3001.sinks.k1.hdfs.rollSize = 134217728
a3001.sinks.k1.hdfs.rollInterval = 600
a3001.sinks.k1.hdfs.batchSize = 10000
a3001.sinks.k1.hdfs.threadsPoolSize = 10
a3001.sinks.k1.hdfs.idleTimeount = 0
a3001.sinks.k1.hdfs.minBlockReplicas = 1

# properties of c1
a3001.channels.c1.type = file
a3001.channels.c1.maxFileSize = 67108864
a3001.channels.c1.transactionCapacity = 10000
a3001.channels.c1.capacity = 50000
a3001.channels.c1.dataDirs = /data/flume/data/channelsData/data
a3001.channels.c1.checkpointDir = /data/flume/data/channelsData/checkpoint

# properties of s1,k1's channel
a3001.sources.s1.channels = c1
a3001.sinks.k1.channel = c1

新增两个文件

6、启动flume

使用nohup启动挂起flume：(PC3/PC2分别启动，注意修改配置文件名)

PC3启动：
nohup bin/flume-ng agent --conf conf --conf-file conf/avro-hdfs.properties --name a3002 -Dflume.root.logger=DEBUG,console -Dorg.apache.flume.log.printconfig=true -Dorg.apache.flume.log.rawdata=true > hdfs.out &

nohup bin/flume-ng agent --conf conf --conf-file conf/avro-a3002.properties --name a3002 -Dflume.root.logger=DEBUG,console -Dorg.apache.flume.log.printconfig=true -Dorg.apache.flume.log.rawdata=true > avro.out &

PC2启动：
nohup bin/flume-ng agent --conf conf --conf-file conf/avro-hdfs.properties --name a3001 -Dflume.root.logger=DEBUG,console -Dorg.apache.flume.log.printconfig=true -Dorg.apache.flume.log.rawdata=true > hdfs.out &

nohup bin/flume-ng agent --conf conf --conf-file conf/avro-a3001.properties --name a3001 -Dflume.root.logger=DEBUG,console -Dorg.apache.flume.log.printconfig=true -Dorg.apache.flume.log.rawdata=true > avro.out &

启动，jps查询，有一个Application的进程