安装配置flume

1. 安装jdk

下载
wget http://apache.dataguru.cn/flume/1.5.2/apache-flume-1.5.2-bin.tar.gz

2. 安装
tar zxvf apache-flume-1.5.2-bin.tar.gz
mv apache-flume-1.5.2-bin /usr/local/flume
cd /usr/local/flume/conf
cp flume-env.sh.template flume-env.sh

测试安装成功
/usr/local/flume/bin/flume-ng version


3. 修改agent配置文件
vim /usr/local/flume/conf/nginx_agent.conf
添加
# Name the components on this agent
a1.sources = r1
a1.sinks = k1
a1.channels = c1

# Describe/configure the source
a1.sources.r1.type = exec
a1.sources.r1.command = tail -F /ext_logs/nginx/kt239api.log     #按照文件名进行跟踪

# Describe the sink
a1.sinks.k1.type = avro
a1.sinks.k1.hostname = 10.2.0.71
a1.sinks.k1.port = 10001

# Use a channel which buffers events in memory
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100

# Bind the source and sink to the channel
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1


启动agent
/usr/local/flume/bin/flume-ng agent --conf /usr/local/flume/conf --conf-file /usr/local/flume/conf/nginx_agent.conf --name a1 -Dflume.root.logger=INFO,console
后台启动
nohup /usr/local/flume/bin/flume-ng agent --conf /usr/local/flume/conf --conf-file /usr/local/flume/conf/nginx_agent.conf --name a1 -Dflume.root.logger=INFO &>> /var/log/flume_nginx.log &


4. 修改collection配置文件
vim /usr/local/flume/conf/nginx_collection.conf
添加
# Name the components on this agent
a1.sources = r1
a1.sinks = k1
a1.channels = c1

# Describe/configure the source
a1.sources.r1.type = avro
a1.sources.r1.bind = 10.2.0.71
a1.sources.r1.port = 10001

# Describe the sink
#a1.sinks.k1.type = logger
a1.sinks.k1.type = elasticsearch
a1.sinks.k1.hostNames = 10.2.0.72:9300
a1.sinks.k1.indexName = nginx_index

# Use a channel which buffers events in memory
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100

# Bind the source and sink to the channel
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1

从elasticsearch的lib目录下拷贝两jar包到flume的lib目录下
elasticsearch-1.2.4.jar
lucene-core-4.8.1.jar

启动collection
/usr/local/flume/bin/flume-ng agent --conf /usr/local/flume/conf --conf-file /usr/local/flume/conf/nginx_collection.conf --name a1 -Dflume.root.logger=INFO,console
后台启动
nohup /usr/local/flume/bin/flume-ng agent --conf /usr/local/flume/conf --conf-file /usr/local/flume/conf/nginx_collection.conf --name a1 -Dflume.root.logger=INFO &>> /var/log/flume_nginx_collection.log &



新开一个窗口
echo "aaaaaaaaaaa" >> /root/nginx.log

回到刚才的窗口就可以看到日志输出



仔细看一下这个页面内容
http://marsorp.iteye.com/blog/1561286