Flume简介

flume概念：
apache Flume 是一个从可以收集例如日志，事件等数据资源，并将这些数量庞大的数据从各项数据资源中集中起来存储的工具/服务，或者数集中机制。flume具有高可用，分布式，配置工具，其设计的原理也是基于将数据流，如日志数据从各种网站服务器上汇集起来存储到HDFS，HBase等集中存储器中。
大数据框架：
数据采集：flume
数据存储：hdfs/hive/hbase
数据计算：mapreduce、hive、sparksql、sparkStreaming、flink
数据可视化：

作用：
主要用来采集数据

flume架构：

source：
数据源
channel：
数据传输通道
sink：
从channel收集数据
event：传输单元
flume数据传输的基本单元，以事件的形式将数据送往目的地

监听端口

->配置文件
cat flume_01.conf
#指定Agent的组件名称（a），一个进程
a.sources=r1
a.channels=c1
a.sinks=k1
a.sources.r1.type=netcat
a.sources.r1.bind=192.168.8.xxx
a.sources.r1.port=12984
a.sources.r1.channels=c1a.channels.c1.type=memory
a.channels.c1.capacity=1000
a.channels.c1.transactionCapacity=1000a.sinks.k1.channel=c1
a.sinks.k1.type=logger运行命令：[root@Master ~]# flume-ng agent --conf conf/ --name a --conf-file ./flume_01.conf -Dflume.root.logger==INFO,console[root@Master ~]# telnet 192.168.8.xx 12984（一台服务）
Trying 192.168.8.xx…
Connected to 192.168.8.xx.
Escape character is ‘^]’.
asdfghjk
OK另一台服务器：telnet 192.168.8.xxx 12984
Trying 192.168.8.xx…
telnet: connect to address 192.168.8.xxx: Connection refused
[root@Slave2 ~]# telnet 192.168.8.xx12984
Trying 192.168.8.xx…
Connected to 192.168.8.xx.
Escape character is ‘^]’.
asdfhjk
OK服务端接受信息;18/12/22 21:22:17 INFO source.NetcatSource: Source starting
18/12/22 21:22:17 INFO source.NetcatSource: Created serverSocket:sun.nio.ch.ServerSocketChannelImpl[/192.168.8.128:12984]
18/12/22 21:22:33 INFO sink.LoggerSink: Event: { headers:{} body: 61 73 64 66 68 6A 6B 0D asdfhjk. }
18/12/22 21:22:38 INFO sink.LoggerSink: Event: { headers:{} body: 61 73 64 66 67 68 6A 6B 0D asdfghjk. }

2)监听文件

-》配置文件
#Describe the agent
a1.sources = r1
a1.sinks = k1
a1.channels = c1# Describe the source
a1.sources.r1.type = exec
a1.sources.r1.channels = c1
a1.sources.r1.command = tail -F /root/flume/test.log
a1.sources.r1.shell = /usr/bin/bash -c# Describe the sink
a1.sinks.k1.type = hdfs
a1.sinks.k1.hdfs.path = hdfs://192.168.8.128:9000/flume/%Y%m%d/%H
a1.sinks.k1.hdfs.filePrefix=logs-
a1.sinks.k1.hdfs.round = true
a1.sinks.k1.hdfs.roundValue = 1
a1.sinks.k1.hdfs.roundUnit = minute
a1.sinks.k1.hdfs.useLocalTimeStamp = true
a1.sinks.k1.hdfs.batchSize = 1000
a1.sinks.k1.hdfs.fileType=DataStream
a1.sinks.k1.hdfs.rollSize=134217700
a1.sinks.k1.hdfs.rollCount=0
a1.sinks.k1.hdfs.minBlockReplicas=1# Use a channel which buffers events in memory
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100# Bind the source and sink to the channel
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1
写入数据 echo “sdfasdfdsa” >test.log
[root@Master flume]# echo “sdfasdfdsa” >test.log
[root@Master flume]# echo “sdfasdfdsa” >>test.log
[root@Master flume]# echo “sdfasdfdsa” >>test.log
[root@Master flume]# echo “sdfasdfdsa” >>test.loghdfs获取采集到数据
[root@Master flume]# hdfs dfs -cat /flume/20181222/22/logs-.1545490462400
fghjkl1234678
fghjkl1234678
fghjkl1234678

3)监听文件夹配置文件

#Describe the agent
a1.sources = r1
a1.sinks = k1
a1.channels = c1# Describe/configure the source
a1.sources.r1.type = spooldir
a1.sources.r1.channels = c1
a1.sources.r1.spoolDir =/root/flume/spool
a1.sources.r1.fileHeader = true# Describe the sink
a1.sinks.k1.type = hdfs
a1.sinks.k1.hdfs.path = hdfs://192.168.8.128:9000/flume/%Y%m%d/%H
a1.sinks.k1.hdfs.filePrefix=spooldir-
a1.sinks.k1.hdfs.round = true
a1.sinks.k1.hdfs.roundValue = 1
a1.sinks.k1.hdfs.roundUnit = second
a1.sinks.k1.hdfs.useLocalTimeStamp = true
a1.sinks.k1.hdfs.batchSize = 1000
a1.sinks.k1.hdfs.fileType=DataStream
a1.sinks.k1.hdfs.rollSize=134217700
a1.sinks.k1.hdfs.rollCount=0
a1.sinks.k1.hdfs.minBlockReplicas=1
#Use a channel which buffers events in memory
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100# Bind the source and sink to the channel
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1启动命令flume-ng agent --conf conf/ --name a1 --conf-file /opt/flume-1.8.0/conf/exec.conf -Dflume.root.logger==INFO,console查询结果

4）多个channel和多个sink