Flink 同步kafka 数据写入hbase

最新推荐文章于 2024-08-06 10:03:20 发布

小小聪

最新推荐文章于 2024-08-06 10:03:20 发布

阅读量1.3k

点赞数

分类专栏： flink kafka

本文链接：https://blog.csdn.net/servletwjx/article/details/89924523

版权

本文介绍如何使用Apache Flink从Kafka实时消费数据，并将这些数据高效地写入到HBase数据库中，实现大数据的实时处理和存储。

摘要由CSDN通过智能技术生成

package com.sitesh.test;
import java.io.*;
import java.util.Arrays;
import java.util.List;
import java.util.Properties;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.state.filesystem.FsStateBackend;
import org.apache.flink.streaming.api.CheckpointingMode;
import org.apache.flink.streaming.api.TimeCharacteristic;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.CheckpointConfig;
import org.apache.flink.streaming.api.environment.CheckpointConfig.ExternalizedCheckpointCleanup;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer010;
import org.apache.flink.util.Collector;
import org.apache.kafka.common.serialization.StringDeserializer;

public class KafkaToHbaseMain {
    private static final Properties props = new Properties();
    //初始化配置文件
    public static void init(String path)
            throws Exception
    {
        InputStream input = new FileInputStream(new File(path));
        try
        {
            props.load(input);
        }
        catch (IOException e)
        {
            e.printStackTrace();
        }
        finally
        {
            input.close();
        }
    }

    public static final class LineSplitter
            implements FlatMapFunction<String, Tuple2<String, Integer>>
    {
        private static final long serialVersionUID = 1L;

        public void flatMap(String value, Collector<Tuple2<String, Integer>> out)
        {
            String[] tokens = value.toLowerCase().split("\\W+");
            for (String