flume 1.8

最新推荐文章于 2022-03-03 10:12:25 发布

weixin_33739541

最新推荐文章于 2022-03-03 10:12:25 发布

阅读量65

点赞数

文章标签：大数据 python json

原文链接：https://my.oschina.net/corleone/blog/1553948

版权

2019独角兽企业重金招聘Python工程师标准>>>

dn5，dn6，dn7 的 flume1.8 安装在 /usr/local/ 目录下

dn5 ： /tmp/example.conf

# example.conf: A single-node Flume configuration

# Name the components on this agent
a1.sources = r1
a1.sinks = k1
a1.channels = c1


####################################
# r1 start 
####################################

a1.sources.r1.type = netcat
a1.sources.r1.bind = localhost
a1.sources.r1.port = 44444

####################################
# c1 start 
####################################

# Use a channel which buffers events in memory
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100

# Describe the sink
a1.sinks.k1.type = logger

# Bind the source and sink to the channel
a1.sinks.k1.channel = c1

####################################
# c1 end 
####################################


a1.sources.r1.channels = c1

# 在dn5 启动 agent ，监听端口为44444
/usr/local/flume/bin/flume-ng agent \
--conf /usr/local/flume/conf \
--conf-file /tmp/example.conf \
--name a1 \
-Dflume.root.logger=INFO,console \
-Dflume.monitoring.port=34545

在同节点另起一个控制台

nc localhost 44444

################################################

flume 1.8 新支持系统参数案例： dn5 ： /tmp/example2.conf

保存在hostname为dn5的 节点上
# example2.conf: A single-node Flume configuration

# Name the components on this agent
a1.sources = r1
a1.sinks = k1
a1.channels = c1


####################################
# r1 start 
####################################

# Describe/configure the source
a1.sources.r1.type = netcat
# 参数化，避免使用 "." 
a1.sources.r1.bind = ${bind_host}
a1.sources.r1.port = ${bind_port}


####################################
# c1 start 
####################################

# Use a channel which buffers events in memory
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100

# Describe the sink
a1.sinks.k1.type = logger

a1.sinks.k1.channel = c1

####################################
# c1 end 
####################################


# Bind the source and sink to the channel
a1.sources.r1.channels = c1

在同节点启动控制台

# 启动 agent ，监听端口为44444
bind_host=dn5 \
bind_port=11111 \
/usr/local/flume/bin/flume-ng agent \
--conf /usr/local/flume/conf \
--conf-file /tmp/example2.conf \
--name a1 \
-Dflume.root.logger=INFO,console \
-DpropertiesImplementation=org.apache.flume.node.EnvVarResolverProperties \
-Dflume.monitoring.port=34546

在局域网内的节点启动控制台，可以多个节点同时使用连接

nc dn5 11111

################################################

dn5 nc做source，在 dn6的logger

# 存在/tmp/ 目录下
# example3.conf: A single-node Flume configuration

# Name the components on this agent
a1.sources = r1
a1.channels = c1 c2
a1.sinks = k1 k2


####################################
# r1 start 
####################################

# source start 
# Describe/configure the source
a1.sources.r1.type = netcat
# 参数化，避免使用 "." 
a1.sources.r1.bind = ${bind_host}
a1.sources.r1.port = ${bind_port}

####################################
# c1 start 
####################################

# channel1 start 

a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100

a1.sinks.k1.type = logger

a1.sinks.k1.channel = c1


####################################
# c1 end 
####################################

####################################
# c2 start 
####################################

a1.channels.c2.type = memory
a1.channels.c2.capacity = 1000
a1.channels.c2.transactionCapacity = 100

a1.sinks.k2.type = avro
a1.sinks.k2.hostname = ${k2_avro_host}
a1.sinks.k2.port = ${k2_avro_port}

a1.sinks.k2.channel=c2


####################################
# c2 end 
####################################


a1.sources.r1.channels = c1 c2

# 在dn5中，启动如下命令
k2_avro_host=dn6 \
k2_avro_port=12345 \
bind_host=dn5 \
bind_port=11111 \
/usr/local/flume/bin/flume-ng agent \
--conf /usr/local/flume/conf \
--conf-file /tmp/example3.conf \
--name a1 \
-Dflume.root.logger=INFO,console \
-DpropertiesImplementation=org.apache.flume.node.EnvVarResolverProperties \
-Dflume.monitoring.port=34547

在dn6中的/tmp/ 下保存如下文件

# example4.conf: A single-node Flume configuration

# Name the components on this agent
a1.sources = r1
a1.sinks = k1
a1.channels = c1


####################################
# r1 start 
####################################

a1.sources.r1.type = avro
a1.sources.r1.bind = ${k2_avro_host}
a1.sources.r1.port = ${k2_avro_port}

####################################
# c1 start 
####################################

# Use a channel which buffers events in memory
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100

# Describe the sink
a1.sinks.k1.type = logger

# Bind the source and sink to the channel
a1.sinks.k1.channel = c1

####################################
# c1 end 
####################################


a1.sources.r1.channels = c1

# 在dn6中 执行如下命令
k2_avro_host=dn6 \
k2_avro_port=12345 \
/usr/local/flume/bin/flume-ng agent \
--conf /usr/local/flume/conf \
--conf-file /tmp/example4.conf \
--name a1 \
-Dflume.root.logger=INFO,console \
-DpropertiesImplementation=org.apache.flume.node.EnvVarResolverProperties \
-Dflume.monitoring.port=34549

# 在局域网内执行如下命令：
nc dn5 11111

source 为taildir

# 
# example5.conf: 
# use taildir source

# Name the components on this agent
a1.sources = r1
a1.channels = c1 c2
a1.sinks = k1 k2

####################################
# r1 start 
####################################

# source start 
# Describe/configure the source
a1.sources.r1.type = TAILDIR
a1.sources.r1.filegroups=fg1 fg2 fg3
a1.sources.r1.filegroups.fg1=/mnt/hgfs/share_dir/fg1/.*\.ndf
a1.sources.r1.filegroups.fg2=/mnt/hgfs/share_dir/fg2/b.ndf
a1.sources.r1.filegroups.fg3=/mnt/hgfs/share_dir/fg2/.*\.ndf

# a1.sources.r1.positionFile=~/.flume/example5.taildir_position.json

####################################
# c1 start 
####################################

# channel1 start 

a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100

a1.sinks.k1.type = logger

a1.sinks.k1.channel = c1

####################################
# c1 end 
####################################


####################################
# c2 start 
####################################

a1.channels.c2.type = memory
a1.channels.c2.capacity = 1000
a1.channels.c2.transactionCapacity = 100

a1.sinks.k2.type = avro
a1.sinks.k2.hostname = ${k2_avro_host}
a1.sinks.k2.port = ${k2_avro_port}

a1.sinks.k2.channel=c2


####################################
# c2 end 
####################################

a1.sources.r1.channels = c1 c2

转载于:https://my.oschina.net/corleone/blog/1553948