Flink自定义Sink写入到HBase

import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Table;

import java.nio.charset.StandardCharsets;

/**
 * Author:panghu
 * Date:2022-05-29
 * Description: 自定义source写入HBase
 */
public class _19SinkCustomtoHBase {
    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(1);

        DataStreamSource<String> streamSource = env.fromElements("hello", "world");

        streamSource.addSink(
                new RichSinkFunction<String>() {
                    //管理HBase的配置信息
                    //因为类重名的问题,将类的完整路径导入
                    org.apache.hadoop.conf.Configuration configuration;
                    Connection connection;

                    @Override
                    public void open(Configuration parameters) throws Exception {
                        super.open(parameters);
                        // HBase的链接参数
                        configuration = HBaseConfiguration.create();
                        configuration.set("hbase.zookeeper.quorum", "hadoop102:2181");
                        connection = ConnectionFactory.createConnection(configuration);
                    }

                    @Override
                    public void close() throws Exception {
                        super.close();
                        connection.close(); //关闭链接
                    }

                    @Override
                    public void invoke(String value, Context context) throws Exception {
                        Table tb = connection.getTable(TableName.valueOf("test")); // 表名
                        Put put = new Put("rowkey".getBytes(StandardCharsets.UTF_8)); // 指定rowkey

                        put.addColumn("info".getBytes(StandardCharsets.UTF_8),   // 列名
                                value.getBytes(StandardCharsets.UTF_8),     // 写入的数据
                                "1".getBytes(StandardCharsets.UTF_8)        // 写入的数据
                        );
                        // 执行put操作
                        tb.put(put);
                        // 关闭表
                        tb.close();
                    }
                }
        );

        env.execute();
    }
}

  • 1
    点赞
  • 6
    收藏
    觉得还不错? 一键收藏
  • 0
    评论
如果你想使用 Flink 批量将数据写入 HBase,可以自定义一个 HBaseSinkFunction。下面是一个简单的示例: ```java public class HBaseBatchSinkFunction extends RichSinkFunction<List<Tuple2<String, String>>> { private transient Connection connection; private transient BufferedMutator bufferedMutator; @Override public void open(Configuration parameters) throws Exception { Configuration config = HBaseConfiguration.create(); config.set("hbase.zookeeper.quorum", "localhost"); config.set("hbase.zookeeper.property.clientPort", "2181"); config.set("zookeeper.znode.parent", "/hbase"); config.set("hbase.client.write.buffer", "10000000"); config.set("hbase.client.retries.number", "3"); connection = ConnectionFactory.createConnection(config); TableName tableName = TableName.valueOf("my_table"); BufferedMutatorParams params = new BufferedMutatorParams(tableName); params.writeBufferSize(1024 * 1024); bufferedMutator = connection.getBufferedMutator(params); } @Override public void invoke(List<Tuple2<String, String>> values, Context context) throws Exception { List<Put> puts = new ArrayList<>(); for (Tuple2<String, String> value : values) { Put put = new Put(Bytes.toBytes(value.f0)); put.addColumn(Bytes.toBytes("my_cf"), Bytes.toBytes("my_col"), Bytes.toBytes(value.f1)); puts.add(put); } bufferedMutator.mutate(puts); } @Override public void close() throws Exception { if (bufferedMutator != null) { bufferedMutator.flush(); bufferedMutator.close(); } if (connection != null) { connection.close(); } } } ``` 在这个自定义HBaseSinkFunction 中,我们使用 BufferedMutator 批量写入数据。在 open() 方法中,我们获取 HBase 连接和缓冲器。在 invoke() 方法中,我们将数据转换为 Put 对象,并添加到缓冲器中。最后,在 close() 方法中,我们刷新缓冲器并关闭连接。 在你的 Flink 程序中,你可以使用这个自定义HBaseSinkFunction,例如: ```java DataStream<Tuple2<String, String>> dataStream = ...; dataStream.addSink(new HBaseBatchSinkFunction()); ``` 这样,你就可以批量将数据写入 HBase 了。
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值