1\在h16上安装tengine成功,目录:/opt/com/soft/tengine-2.1.0/,h15上安装flume成功,目录:/home/apache-flume-1.6.0-bin
2\修改conf/nginx.conf文件
#user nobody;
worker_processes 1;
events {
worker_connections 1024;
}
http {
include mime.types;
default_type application/octet-stream;
log_format my_format '$remote_addr^A$msec^A$http_host^A$request_uri'; //自定义格式,二进制分割符--》 ^A 方便数据分析
sendfile on;
keepalive_timeout 65;
server {
listen 80;
server_name localhost;
location = /log.jpg {
access_log /opt/flume/flume.log my_format; //指定在页面访问:”192.168.142.115/log.jpg”写入的日志文件目录
root html;
}
error_page 500 502 503 504 /50x.html;
location = /50x.html {
root html;
}
}
}
3\启动nginx,部署项目
#server nginx start
部署项目:
在eclipse上创建项目--》first_project(包含js的sdk)
4\页面访问:192.168.142.116:8080/demo.jsp
5\如果有日志写入/opt/flume/flume.log,那么成功!
6\编写代码,实现通过调用flume的jar写入信息到hdfs中
(1)导入flume的所有jar包
(2)编写java类:FlumeTest.java
package com.com.test;
import java.nio.charset.Charset;
import java.util.Random;
import org.apache.flume.Event;
import org.apache.flume.EventDeliveryException;
import org.apache.flume.api.RpcClient;
import org.apache.flume.api.RpcClientFactory;
import org.apache.flume.event.EventBuilder;
public class FlumeTest {
public static void main(String[] args) {
MyRpcClientFacade client = new MyRpcClientFacade();
client.init("192.168.142.115", 41414); //flume所在的机器ip,testflume5配置的java端口号
for (int i = 0; i < 13; i++) {
String sampleData = "Hello Flume!"+new Random().nextInt(220);
client.sendDataToFlume(sampleData);
}
client.cleanUp();
}
}
class MyRpcClientFacade {
private RpcClient client;
private String hostname;
private int port;
public void init(String hostname, int port) {
this.hostname = hostname;
this.port = port;
this.client = RpcClientFactory.getDefaultInstance(hostname, port);
}
public void sendDataToFlume(String data) {
Event event = EventBuilder.withBody(data, Charset.forName("UTF-8"));
try {
client.append(event);
} catch (EventDeliveryException e) {
client.close();
client = null;
client = RpcClientFactory.getDefaultInstance(hostname, port);
}
}
public void cleanUp() {
client.close();
}
}
(3)编写testflume5文件(目录:在h15中的/home/test/ 下)
内容如下:
# example.conf: A single-node Flume configuration
# Name the components on this agent
a1.sources = r1
a1.sinks = k1
a1.channels = c1
# Describe/configure the source
a1.sources.r1.type =avro
a1.sources.r1.bind=h15
a1.sources.r1.port=41414
# Describe the sink
a1.sinks.k1.type =hdfs
a1.sinks.k1.hdfs.rollInterval=3
a1.sinks.k1.hdfs.rollSize=0
a1.sinks.k1.hdfs.rollCount=0
a1.sinks.k1.hdfs.path=hdfs://yangjifei/flume/data/%Y-%m-%d
a1.sinks.k1.hdfs.useLocalTimeStamp=true
a1.sinks.k1.hdfs.fileType=DataStream
a1.sinks.k1.hdfs.idleTimeout=3
# Use a channel which buffers events in memory
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100
# Bind the source and sink to the channel
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1
(4)在h15上执行以下代码,使得flume处于等待状态
#flume-ng agent --conf /home/apache-flume-1.6.0-bin/conf --conf-file /home/test/testflume5 --name a1 -Dflume.root.logger=INFO,console &
(5)执行main类:FlumeTest.java
查看hdfs文件系统目录:/flume/data/%Y-%m-%d,是否生成文件即可。