范例主要的处理流程说明如下:
1、web应用的日志信息实时输出到ActiveMQ
2、Flume的Source从MQ指定队列中获取消息,并提交到内存型Channel中
3、自定义Sink从Channel提取event,对event进行转换处理,并写入到Oracle数据库
开发自定义的Sink,编译打包成jar包,并上传到/opt/apache-flume-1.6.0-bin/plugins.d/flume-oracle-sink/lib,自定义sink的存放目录规范请查阅官方相关文档。
自定义Sink的代码骨架如下:
public class OracleSink extends AbstractSink implements Configurable {
private int batchSize;
private String charset;
public void configure(Context context) {
this.batchSize = context.getInteger(OracleSinkConfiguration.BATCH_SIZE, new Integer(OracleSinkConfiguration.BATCH_SIZE_DEFAULT)).intValue();
this.charset = context.getString(OracleSinkConfiguration.CHARSET, OracleSinkConfiguration.CHARSET_DEFAULT);
}
public synchronized void start() {
super.start();
}
public Status process() throws EventDeliveryException {
Sink.Status result = Sink.Status.READY;
Channel channel = getChannel();
Transaction transaction = null;
Event event = null;
try{
transaction = channel.getTransaction();
transaction.begin();
for(int i=0; i<this.batchSize; i++){
event = channel.take();
//Map headers = event.getHeaders();
if(event == null){
break;
}else{
byte[] eventBody = event.getBody();
String bodyString = new String(eventBody, this.charset);
//something code here
}
}
transaction.commit();
} catch (Exception ex) {
System.out.println(ex.toString());x
result = Sink.Status.BACKOFF;
if (transaction != null) {
try {
transaction.rollback();
} catch (Exception e) {
throw Throwables.propagate(e);
}
}
} finally {
if (transaction != null) {
transaction.close();
}
}
return result;
}
public synchronized void stop() {
super.stop();
}
}
在Flume根目录的config文件夹内新建一个flume config文件,内容如下:
# Name the components on this agent agent.sources = source1 agent.sinks = sink1 sink2 agent.channels = channel1 #source agent.sources.source1.type = jms agent.sources.source1.initialContextFactory = org.apache.activemq.jndi.ActiveMQInitialContextFactory agent.sources.source1.connectionFactory = ConnectionFactory agent.sources.source1.providerURL = failover://(tcp://192.168.247.2:61616?tcpNoDelay=true) agent.sources.source1.destinationType = QUEUE agent.sources.source1.destinationName = my_queue agent.sources.source1.batchSize = 200 agent.sources.source1.pollTimeout = 2000 #channel #agent.channels.channel1.type = file #agent.channels.channel1.checkpointDir = /tmp/flume/loadcheckpoint #agent.channels.channel1.dataDirs = /tmp/flume/loaddata #agent1.channels.channel1.capacity = 1000 #agent1.channels.channel1.transactionCapactiy = 100 agent.channels.channel1.type = memory agent.channels.channel1.capacity = 10000 agent.channels.channel1.transactionCapacity = 200 #sinkgroups agent.sinkgroups = g1 #agent.sinkgroups.g1.sinks = sink1 sink2 #agent.sinkgroups.g1.processor.type = failover #agent.sinkgroups.g1.processor.priority.sink1 = 10 #agent.sinkgroups.g1.processor.priority.sink2 = 5 #agent.sinkgroups.g1.processor.maxpenalty = 10000 agent.sinkgroups.g1.sinks = sink1 sink2 agent.sinkgroups.g1.processor.type = load_balance agent.sinkgroups.g1.processor.backoff = true agent.sinkgroups.g1.processor.selector = round_robin # Describe the sink agent.sinks.sink1.type = com.cjm.flume.oraclesink.OracleSink agent.sinks.sink1.username = cjm agent.sinks.sink1.password = 111 agent.sinks.sink2.type = com.cjm.flume.oraclesink.OracleSink agent.sinks.sink2.username = cjm agent.sinks.sink2.password = 222 agent.sinks.sink2.batchSize= 100 agent.sinks.sink2.charset = UTF-8 # Bind the source and sink to the channel agent.sources.source1.channels = channel1 agent.sinks.sink1.channel = channel1 agent.sinks.sink2.channel = channel1
启动ActiveMQ
启动Flume
cd /opt/apache-flume-1.6.0-bin
bin/flume-ng agent -c conf -f conf/flume-agent.conf -n agent -Dflume.root.logger=INFO,console
后台服务方式启动 nohup bin/flume-ng agent -c conf -f conf/flume-agent.conf -n agent >flume.log &
模拟发送信息到MQ
public class UMProducer {
private MessageProducer producer = null;
private Connection connection = null;
private Session session = null;
private String data = null;
public void start(){
try{
InputStream in = UMProducer.class.getResourceAsStream("/data.txt");
this.data = IOUtils.toString(in);
ConnectionFactory connectionFactory = new ActiveMQConnectionFactory(
ActiveMQConnection.DEFAULT_USER,
ActiveMQConnection.DEFAULT_PASSWORD,
"failover://(tcp://192.168.247.2:61616?tcpNoDelay=true)");
connection = connectionFactory.createConnection();
((ActiveMQConnection)connection).setUseAsyncSend(true);
connection.start();
session = connection.createSession(Boolean.TRUE, Session.CLIENT_ACKNOWLEDGE);
Destination destination = session.createQueue("my_queue");
producer = session.createProducer(destination);
producer.setDeliveryMode(DeliveryMode.NON_PERSISTENT);
producer.setTimeToLive(1000 * 60 * 60 * 24);
while(true){
TextMessage message = session.createTextMessage(this.data);
producer.send(message);
}
}catch(Exception ex){
ex.printStackTrace();
}
}
public void stop(){
try{
if(session != null) {
session.close();
}
if(connection != null) {
connection.close();
}
if(producer != null){
producer.close();
}
}catch(Exception ex){
ex.printStackTrace();
}
}
public static void main(String[] args) {
UMProducer producer = new UMProducer();
producer.start();
}
}