flume安装使用笔记

(base) [root@lijia1 install]# tar -zxf flume-ng-1.6.0-cdh5.14.0.tar.gz -C …/bigdata/
(base) [root@lijia1 bigdata]# mv apache-flume-1.6.0-cdh5.14.0-bin/ flume160514
(base) [root@lijia1 bigdata]# cd ./flume160514/
(base) [root@lijia1 flume160514]# cd ./conf/
(base) [root@lijia1 conf]# cp flume-env.sh.template flume-env.sh
(base) [root@lijia1 conf]# vi ./flume-env.sh
export JAVA_HOME=/opt/bigdata/jdk180/
export JAVA_OPTS="-Xms2000m -Xmx2000m -Dcom.sun.management.jmxremote"

(base) [root@lijia1 conf]# mkdir job

(base) [root@lijia1 job]# yum install -y nc
(base) [root@lijia1 job]# nc -lk 44444

[root@lijia1 ~]# yum list telnet* 列出telnet相关的安装包
[root@lijia1 ~]# yum install telnet-server 安装telnet服务
[root@lijia1 ~]# yum install telnet.* 安装telnet客户端

[root@lijia1 ~]# telnet localhost 44444

// 读取电脑指定端口44444内容
[root@lijia1 job]# vi netcat-flume-logger.conf
a1.sources = r1
a1.sinks = k1
a1.channels = c1

a1.sources.r1.type = netcat
a1.sources.r1.bind = localhost
a1.sources.r1.port = 44444

a1.sinks.k1.type = logger

a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100

a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1

[root@lijia1 flume160514]# ./bin/flume-ng agent --name a1 --conf conf/ --conf-file conf/job/netcat-flume-logger.conf -Dflume.root.logger=INFO,console

// 读取指定文件内容
[root@lijia1 job]# vi file-flume-logger.conf
a2.sources = r1
a2.sinks = k1
a2.channels = c1

a2.sources.r1.type = exec
a2.sources.r1.command = tail -f /opt/bigdata/flume160514/conf/job/tmp.txt

a2.sinks.k1.type = logger

a2.channels.c1.type = memory
a2.channels.c1.capacity = 1000
a2.channels.c1.transactionCapacity = 100

a2.sources.r1.channels = c1
a2.sinks.k1.channel = c1

[root@lijia1 flume160514]# ./bin/flume-ng agent --name a2 --conf conf/ --conf-file conf/job/file-flume-logger.conf -Dflume.root.logger=INFO,console

[root@lijia1 prodata]# wc -l events.csv 查看文件总行数

// 读取指定文件夹中复合正则表达式的文件内容
[root@lijia1 job]# vi ./events-flume-logger.conf
events.sources = eventsSource
events.channels = eventsChannel
events.sinks = eventsSink

events.sinks.eventsSink.type = logger

events.sources.eventsSource.type = spooldir
events.sources.eventsSource.spoolDir = /opt/kb07file/flumeFile/events
events.sources.eventsSource.deserializer = LINE
events.sources.eventsSource.deserializer.maxLineLength = 32000
events.sources.eventsSource.includePattern = events_[0-9]{4}-[0-9]{2}-[0-9]{2}.csv

events.channels.eventsChannel.type = file
events.channels.eventsChannel.checkpointDir = /opt/kb07file/flumeFile/checkpoint/events
events.channels.eventsChannel.dataDirs = /opt/kb07file/flumeFile/data/events

events.sources.eventsSource.channels = eventsChannel
events.sinks.eventsSink.channel = eventsChannel

[root@lijia1 flume160514]# ./bin/flume-ng agent --name events --conf conf/ --conf-file conf/job/events-flume-logger.conf -Dflume.root.logger=INFO,console