Flume(7) flume自定义Sinks实现

源码地址

前言 接上一篇Flume(6) flume自定义Sources实现，我们总结了一下自定义source的流程，这次我们实现一个自己的Sink，将数据Sink到Mysql数据库中。

创建数据库相关

CREATE TABLE `flume` (
  `content` varchar(255) DEFAULT NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;

引入依赖

<properties>
		<version.flume>1.9.0</version.flume>
	</properties>
	
	<dependencies>

		<dependency>
			<groupId>org.apache.flume</groupId>
			<artifactId>flume-ng-core</artifactId>
			<version>${version.flume}</version>
		</dependency>
		<dependency>
			<groupId>org.apache.flume</groupId>
			<artifactId>flume-ng-configuration</artifactId>
			<version>${version.flume}</version>
		</dependency>

		<dependency>
			<groupId>mysql</groupId>
			<artifactId>mysql-connector-java</artifactId>
			<version>5.1.47</version>
		</dependency>
		
</dependencies>

编写自定义MysqlSink

/**
 * @Author: ynz
 * @Date: 2019/5/7/007 16:38
 * @Version 1.0
 */
public class MysqlSink extends AbstractSink implements Configurable {

    private static final Logger logger = LoggerFactory.getLogger(MysqlSink.class);
    private Connection connect;
    private Statement stmt;
    private String columnName;
    private String url;
    private String user;
    private String password;
    private String tableName;

    // 在整个sink结束时执行一遍
    @Override
    public synchronized void stop() {
        // TODO Auto-generated method stub
        super.stop();
    }

    // 在整个sink开始时执行一遍，用来初始化数据库连接
    @Override
    public synchronized void start() {
        // TODO Auto-generated method stub
        super.start();
        try {
            connect = DriverManager.getConnection(url, user, password);
            // 连接URL为 jdbc:mysql//服务器地址/数据库名 ，后面的2个参数分别是登陆用户名和密码
            stmt = connect.createStatement();
        } catch (SQLException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }

    }

    @Override
    public Status process() throws EventDeliveryException {
        // TODO Auto-generated method stub
        //事务，获取event什么的都是模板。仿照别的sink写就OK
        Channel ch = getChannel();
        Transaction txn = ch.getTransaction();
        Event event = null;
        txn.begin();
        while (true) {
            event = ch.take();
            if (event != null) {
                break;
            }
        }
        try {
            String rawbody = new String(event.getBody());
            logger.info("sink:{}",rawbody);
            String body = rawbody;
            if (body.split(",").length == columnName.split(",").length) {
                String sql = "insert into " + tableName + "(" + columnName + ") values('" + body + "')";
                logger.info("sql:{}",sql);
                //logger.error("sql:"+sql);
                stmt.executeUpdate(sql);
                txn.commit();
                return Status.READY;
            } else {
                txn.rollback();
                return null;
            }
        } catch (Throwable th) {
            txn.rollback();

            if (th instanceof Error) {
                throw (Error) th;
            } else {
                throw new EventDeliveryException(th);
            }
        } finally {
            txn.close();
        }
    }

    @Override
    public void configure(Context context) {
        columnName = context.getString("column_name");
        Preconditions.checkNotNull(columnName, "column_name must be set!!");
        url = context.getString("url");
        Preconditions.checkNotNull(url, "url must be set!!");
        user = context.getString("user");
        Preconditions.checkNotNull(user, "user must be set!!");
        //我的mysql没有密码。所以这里不检查密码为空
        password = context.getString("password");
        // Preconditions.checkNotNull(password, "password must be set!!");
        tableName = context.getString("tableName");
        Preconditions.checkNotNull(tableName, "tableName must be set!!");
    }
}

将写好的程序打成jar包和mysql驱动包，放到flume的lib下。

编写mysql.conf

#定义agent名， source、channel、sink的名称
a1.sources = r1
a1.channels = c1
a1.sinks = k1

#具体定义source,这里的type是自定义的source的类的全路径
a1.sources.r1.type = cn.myframe.source.TailFileSource
#这里的参数名都和自定义类的参数一直
#读取哪个文件
a1.sources.r1.filePath = /usr/local/flume/data2/a.txt
#偏移量保存的文件
a1.sources.r1.positionFile = /usr/local/flume/data2/index
#时间间隔，每隔多久读取一次
a1.sources.r1.interval = 2000
#编码
a1.sources.r1.charset = UTF-8

#具体定义channel
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100

#具体定义sink
a1.sinks.k1.type = cn.myframe.sink.MysqlSink
a1.sinks.k1.column_name = content
a1.sinks.k1.tableName= flume
a1.sinks.k1.url= jdbc:mysql://10.10.2.137:3306/flume?allowMultiQueries=true&useUnicode=true&characterEncoding=UTF-8
a1.sinks.k1.user= root
a1.sinks.k1.password= Rojao@123


#组装source、channel、sink
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1

启动

 flume-ng agent --conf /usr/local/flume/apache-flume-1.9.0-bin/conf --conf-file  /usr/local/flume/apache-flume-1.9.0-bin/conf/mysql.conf --name a1 -Dflume.root.logger=INFO,console

可以把a.txt中的每一行数据保存到mysql数据库