Apache Flume 实现过滤功能

A、B两台日志服务机器实时生产日志主要类型为access.log、nginx.log、web.log
现在要求把A、B 机器中的access.log、nginx.log、web.log 采集汇总到C机器上然后统一收集到hdfs中,并且在hdfs中输出的目录指定为:

/source/logs/access/20180101/**
/source/logs/nginx/20180101/**
/source/logs/web/20180101/**

场景图
在这里插入图片描述
数据流程处理图
在这里插入图片描述

服务器A对应的IP为 192.168.100.201
服务器B对应的IP为 192.168.100.202
服务器C对应的IP为 192.168.100.203

采集端配置文件

hadoop01与hadoop02服务器开发flume的配置文件

cd /export/servers/apache-flume-1.8.0-bin/tmpconf
vim exec_source_avro_sink.conf
# Name the components on this agent
a1.sources = r1 r2 r3
a1.sinks = k1
a1.channels = c1

# Describe/configure the source
a1.sources.r1.type = exec
a1.sources.r1.command = tail -F /home/taillogs/access.log
a1.sources.r1.interceptors = i1
a1.sources.r1.interceptors.i1.type = static
##  static拦截器的功能就是往采集到的数据的header中插入自己定义的key-value对
a1.sources.r1.interceptors.i1.key = type
a1.sources.r1.interceptors.i1.value = access

a1.sources.r2.type = exec
a1.sources.r2.command = tail -F /home/taillogs/nginx.log
a1.sources.r2.interceptors = i2
a1.sources.r2.interceptors.i2.type = static
a1.sources.r2.interceptors.i2.key = type
a1.sources.r2.interceptors.i2.value = nginx

a1.sources.r3.type = exec
a1.sources.r3.command = tail -F /home/taillogs/web.log
a1.sources.r3.interceptors = i3
a1.sources.r3.interceptors.i3.type = static
a1.sources.r3.interceptors.i3.key = type
a1.sources.r3.interceptors.i3.value = web

# Use a channel which buffers events in memory
a1.channels.c1.type = memory
a1.channels.c1.capacity = 20000
a1.channels.c1.transactionCapacity = 10000

# Describe the sink
a1.sinks.k1.type = avro
a1.sinks.k1.hostname = hadoop03
a1.sinks.k1.port = 41414

# Bind the source and sink to the channel
a1.sources.r1.channels = c1
a1.sources.r2.channels = c1
a1.sources.r3.channels = c1
a1.sinks.k1.channel = c1

注:

a1.sources.r1.interceptors = i1
a1.sources.r1.interceptors.i1.type = static
##  static拦截器的功能就是往采集到的数据的header中插入自己定## 义的key-value对
a1.sources.r1.interceptors.i1.key = type
a1.sources.r1.interceptors.i1.value = access

服务端配置文件

在hadoop03上面开发flume配置文件

cd /export/servers/apache-flume-1.8.0-bin/tmpconf
vim avro_source_hdfs_sink.conf
a1.sources = r1
a1.sinks = k1
a1.channels = c1
#定义source
a1.sources.r1.type = avro
a1.sources.r1.bind = 192.168.100.203
a1.sources.r1.port =41414

#添加时间拦截器
a1.sources.r1.interceptors = i1
a1.sources.r1.interceptors.i1.type = org.apache.flume.interceptor.TimestampInterceptor$Builder


#定义channels
a1.channels.c1.type = memory
a1.channels.c1.capacity = 20000
a1.channels.c1.transactionCapacity = 10000

#定义sink
a1.sinks.k1.type = hdfs
a1.sinks.k1.hdfs.path=hdfs://192.168.100.201:8020/source/logs/%{type}/%Y%m%d
 
#组装source、channel、sink
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1

采集端文件生成脚本

在hadoop01与hadoop02上面开发shell脚本,模拟数据生成

cd /home
vim server.sh 
#!/bin/bash
while true
do  
 date >> /home/taillogs/access.log; 
 date >> /home/taillogs/web.log;
 date >> /home/taillogs/nginx.log;
  sleep 0.5;
done

顺序启动服务

hadoop03启动flume实现数据收集

cd /export/servers/apache-flume-1.8.0-bin/
bin/flume-ng agent -c conf -f tmpconf/avro_source_hdfs_sink.conf -name a1 -Dflume.root.logger=DEBUG,console

node01与node02启动flume实现数据监控

cd /export/servers/apache-flume-1.8.0-bin/
bin/flume-ng agent -c conf -f tmpconf/exec_source_avro_sink.conf -name a1 -Dflume.root.logger=DEBUG,console

hadoop01与hadoop02启动生成文件脚本

cd /home
sh server.sh

运行效果:

[root@hadoop03 ~]# hadoop fs -ls /source/logs
Found 3 items
drwxrwxrwx   - root supergroup          0 2019-12-05 17:07 /source/logs/access
drwxrwxrwx   - root supergroup          0 2019-12-05 17:07 /source/logs/nginx
drwxrwxrwx   - root supergroup          0 2019-12-05 17:07 /source/logs/web
[root@hadoop03 ~]# hadoop fs -lsr /source/logs/access
lsr: DEPRECATED: Please use 'ls -R' instead.
drwxrwxrwx   - root supergroup          0 2019-12-05 17:08 /source/logs/access/20191205
-rw-r--r--   2 root supergroup        735 2019-12-05 17:07 /source/logs/access/20191205/FlumeData.1575536820842
-rw-r--r--   2 root supergroup        755 2019-12-05 17:07 /source/logs/access/20191205/FlumeData.1575536820843
-rw-r--r--   2 root supergroup        755 2019-12-05 17:07 /source/logs/access/20191205/FlumeData.1575536820844
-rw-r--r--   2 root supergroup        735 2019-12-05 17:07 /source/logs/access/20191205/FlumeData.1575536820845
-rw-r--r--   2 root supergroup        755 2019-12-05 17:07 /source/logs/access/20191205/FlumeData.1575536820846
-rw-r--r--   2 root supergroup        735 2019-12-05 17:07 /source/logs/access/20191205/FlumeData.1575536820847
-rw-r--r--   2 root supergroup        775 2019-12-05 17:07 /source/logs/access/20191205/FlumeData.1575536820848
-rw-r--r--   2 root supergroup        755 2019-12-05 17:07 /source/logs/access/20191205/FlumeData.1575536820849
-rw-r--r--   2 root supergroup        735 2019-12-05 17:08 /source/logs/access/20191205/FlumeData.1575536820850
-rw-r--r--   2 root supergroup        735 2019-12-05 17:08 /source/logs/access/20191205/FlumeData.1575536820851
-rw-r--r--   2 root supergroup        735 2019-12-05 17:08 /source/logs/access/20191205/FlumeData.1575536820852
-rw-r--r--   2 root supergroup        735 2019-12-05 17:08 /source/logs/access/20191205/FlumeData.1575536820853.tmp
[root@hadoop03 ~]# 
发布了216 篇原创文章 · 获赞 182 · 访问量 7392
展开阅读全文

没有更多推荐了,返回首页

©️2019 CSDN 皮肤主题: 技术黑板 设计师: CSDN官方博客

分享到微信朋友圈

×

扫一扫,手机浏览