自定义flume采集mysql_flume自定义sink之mysql

最新推荐文章于 2021-12-26 21:27:03 发布

weixin_39714565

最新推荐文章于 2021-12-26 21:27:03 发布

阅读量86

点赞数

文章标签：自定义flume采集mysql

本文链接：https://blog.csdn.net/weixin_39714565/article/details/114043921

版权

本文介绍了一个名为 `MySink` 的自定义 Apache Flume 源，该源将接收到的事件数据插入到 MySQL 数据库中。配置包括设置数据列名、数据库 URL、用户名、密码和表名。在启动和停止 sink 时，执行相应的数据库连接和关闭操作。在处理事件过程中，检查数据长度与列名匹配，并执行 SQL 插入语句。整个流程演示了如何使用 Flume 进行数据集成。

摘要由CSDN通过智能技术生成

package me;

import static org.mockito.Matchers.booleanThat;

import java.sql.Connection;

import java.sql.DriverManager;

import java.sql.SQLException;

import java.sql.Statement;

import org.apache.flume.Channel;

import org.apache.flume.Context;

import org.apache.flume.Event;

import org.apache.flume.EventDeliveryException;

import org.apache.flume.Transaction;

import org.apache.flume.conf.Configurable;

import org.apache.flume.sink.AbstractSink;

import com.google.common.base.Preconditions;

public class MySink extends AbstractSink implements Configurable {

private Connection connect;

private Statement stmt;

private String columnName;

private String url;

private String user;

private String password;

private String tableName;

// 在整个sink结束时执行一遍

@Override

public synchronized void stop() {

// TODO Auto-generated method stub

super.stop();

}

// 在整个sink开始时执行一遍

@Override

public synchronized void start() {

// TODO Auto-generated method stub

super.start();

try {

connect = DriverManager.getConnection(url, user, password);

// 连接URL为 jdbc:mysql//服务器地址/数据库名，后面的2个参数分别是登陆用户名和密码

stmt = connect.createStatement();

} catch (SQLException e) {

// TODO Auto-generated catch block

e.printStackTrace();

}

// 不断循环调用

@Override

public Status process() throws EventDeliveryException {

// TODO Auto-generated method stub

Channel ch = getChannel();

Transaction txn = ch.getTransaction();

Event event = null;

txn.begin();

while (true) {

event = ch.take();

if (event != null) {

break;

}

try {

String body = new String(event.getBody());

if (body.split(",").length == columnName.split(",").length) {

String sql = "insert into " + tableName + "(" + columnName + ") values(" + body + ")";

stmt.executeUpdate(sql);

txn.commit();

return Status.READY;

} else {

txn.rollback();

return null;

}

} catch (Throwable th) {

txn.rollback();

if (th instanceof Error) {

throw (Error) th;

} else {

throw new EventDeliveryException(th);

}

} finally {

txn.close();

}

@Override

public void configure(Context arg0) {

columnName = arg0.getString("column_name");

Preconditions.checkNotNull(columnName, "column_name must be set!!");

url = arg0.getString("url");

Preconditions.checkNotNull(url, "url must be set!!");

user = arg0.getString("user");

Preconditions.checkNotNull(user, "user must be set!!");

password = arg0.getString("password");

Preconditions.checkNotNull(password, "password must be set!!");

tableName = arg0.getString("tableName");

Preconditions.checkNotNull(tableName, "tableName must be set!!");

}

agent.sources = s1

agent.channels = c1

agent.sinks = sk1

agent.sources.s1.type = netcat

agent.sources.s1.bind = localhost

agent.sources.s1.port = 5678

agent.sources.s1.channels = c1

agent.sinks.sk1.type = me.MySink

agent.sinks.sk1.url=jdbc:mysql://192.168.16.33:3306/test

agent.sinks.sk1.tableName= test.user

agent.sinks.sk1.user=root

[email protected]

agent.sinks.sk1.column_name=id, username, password

agent.sinks.sk1.channel = c1

agent.channels.c1.type = memory

agent.channels.c1.capacity = 1000

agent.channels.c1.transactionCapacity = 100

lihudeMacBook-Pro:~ SunAndLi$ cd hadoop-2.7.2/flume/

lihudeMacBook-Pro:flume SunAndLi$ bin/flume-ng agent -c conf -f conf/sink-mysql --name agent -Dflume.root.logger=INFO,console

原文：http://www.cnblogs.com/sunyaxue/p/6645415.html