Flink自定义HBasesink的过程

布尔科技技术团队

已于 2022-03-03 13:34:24 修改

阅读量9.8k

点赞数 3

文章标签： flink 大数据 hbase 汽车物联网

于 2022-03-03 13:30:44 首次发布

本文链接：https://blog.csdn.net/BOOLTechnology/article/details/123251446

版权

文末有官方的完整的HBasesink代码;

Flink自定义sink端

自定义sink主要有二种实现方式,一种是实现RichSinkFunction和SInkFuncion

如果要实现Checkpointed还需要实现CheckpointedFunction接口

推荐使用RichSinkFunction全生命周期函数

HBaseSink的实现

简单版实现RichSinkFunction 但不具备通用性,没对一个源就需要修改代码,且没有提供构建方法,开始自己写的一个版本

import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.util.Bytes;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;


/**
 * @author WangKaiYu
 * @date 2022-02-18 15:19
 */

public class HbaseSink extends RichSinkFunction<String>{
    private final static Logger logger = LoggerFactory.getLogger(HbaseSink.class);
    private static org.apache.hadoop.conf.Configuration configuration;
    private static Connection connection= null;
    private static BufferedMutator mutator;
    private static String Tablename="student";

    @Override
    public void open(Configuration parameters) throws Exception {
        configuration=HBaseConfiguration.create();
        configuration.set("hbase.master", "192.168.10.154:16000");
        configuration.set("hbase.zookeeper.quorum", "192.168.10.154,192.168.10.155,192.168.10.196");
        configuration.set("hbase.zookeeper.property.clientPort", "2181");
        try {
//           asyncConnection = ConnectionFactory.createAsyncConnection(configuration);
             connection = ConnectionFactory.createConnection(configuration);
//            connection.getBufferedMutator(TableName.valueOf(Tablename).)
        }catch (Exception e){
            e.printStackTrace();
        }
        BufferedMutatorParams params = new BufferedMutatorParams(TableName.valueOf(Tablename));

        //缓存大小
        params.writeBufferSize(2*1024*1024);
        //最大时间
        params.setWriteBufferPeriodicFlushTimeoutMs(5 * 1000L);
//        AsyncConnection asyncConnection = HbaseSink.asyncConnection.get();
        try {
            mutator = connection.getBufferedMutator(params);
        }catch (IOException e){
            logger.error("当前获取bufferedMutator 失败：" + e.getMessage());
        }

    }

    @Override
    public void close() throws Exception {
        if (mutator != null) {
            mutator.close();
        }
        if (connection != null) {
            connection.close();
        }

    }

    @Override
    public void invoke(String value, Context context) throws Exception {
        String familyName = "info";
        String[] values = value.split(",");
        Put put = new Put(Bytes.toBytes(values[0]));
        put.addColumn(Bytes.toBytes(familyName),Bytes.toBytes("name"),Bytes.toBytes(values[1]));
        put.addColumn(Bytes.toBytes(familyName),Bytes.toBytes("age"),Bytes.toBytes(values[2]));
        mutator.mutate(put);
        // 指定时间内的数据强制刷写到hbase
        mutator.flush();


    }
}