Flink读Kafka写入HBase

Flink从kafka读数据写入HBase

Flink官网帮助文档链接
先导入依赖

<dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-connector-hbase_2.11</artifactId>
            <version>1.11.2</version>
</dependency>

import org.apache.flink.streaming.api.scala._
import org.apache.flink.table.api._
import org.apache.flink.table.api.bridge.scala._


object Demo8ToHbase {
  def main(args: Array[String]): Unit = {
    val bsEnv: StreamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment

    val bsSettings: EnvironmentSettings = EnvironmentSettings
      .newInstance()
      .useBlinkPlanner() //使用blink的计划器
      .inStreamingMode() //使用流模型
      .build()

    //窗口table 环境
    val bsTableEnv: StreamTableEnvironment = StreamTableEnvironment.create(bsEnv, bsSettings)

    /**
     * source表
     */
    bsTableEnv.executeSql(
      """CREATE TABLE canal_student (
        | id STRING,
        | name STRING,
        | age BIGINT,
        | gender STRING,
        | clazz STRING
        |) WITH (
        | 'connector' = 'kafka',
        | 'topic' = 'student.student',
        | 'properties.bootstrap.servers' = 'master:9092',
        | 'properties.group.id' = 'aaa',
        | 'format' = 'canal-json',
        | 'scan.startup.mode' = 'earliest-offset',
        | 'canal-json.ignore-parse-errors'='true'
        |)
      """.stripMargin)

    /**
     * hbase sink表
     * 需要先在HBase里创建表
     * create 'student','info'
     */
    bsTableEnv.executeSql(
      """
        |CREATE TABLE hbase_sink (
        | id INT,
        | info ROW<name STRING,age BIGINT,gender STRING,clazz STRING>,
        | PRIMARY KEY (id) NOT ENFORCED
        |) WITH (
        | 'connector' = 'hbase-1.4',
        | 'table-name' = 'student',
        | 'zookeeper.quorum' = 'master:2181'
        |)
        |""".stripMargin)

    bsTableEnv.executeSql(
      """
        |
        |insert into hbase_sink
        |select id,ROW(name,age,gender,clazz) as info from canal_student
        |
      """.stripMargin)
  }
}

idea运行代码

然后在HBase里查看数据

scan 'student'

读取Hbase数据

package com.liu.sql

import org.apache.flink.streaming.api.scala._
import org.apache.flink.table.api._
import org.apache.flink.table.api.bridge.scala._

object Demo9Readhbase {
  def main(args: Array[String]): Unit = {
    val bsEnv: StreamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment

    val bsSettings: EnvironmentSettings = EnvironmentSettings
      .newInstance()
      .useBlinkPlanner() //使用blink的计划器
      .inStreamingMode() //使用流模型
      .build()

    //窗口table 环境
    val bsTableEnv: StreamTableEnvironment = StreamTableEnvironment.create(bsEnv, bsSettings)

    /**
     * 读取HBase---有界流
     */
    bsTableEnv.executeSql(
      """
        |CREATE TABLE hbase_sink (
        | id INT,
        | info ROW<name STRING,age BIGINT,gender STRING,clazz STRING>,
        | PRIMARY KEY (id) NOT ENFORCED
        |) WITH (
        | 'connector' = 'hbase-1.4',
        | 'table-name' = 'student',
        | 'zookeeper.quorum' = 'master:2181'
        |)
        |""".stripMargin)

    bsTableEnv.executeSql(
      """
        |CREATE TABLE print_table WITH ('connector' = 'print')
        |LIKE hbase_source (EXCLUDING ALL)
        |""".stripMargin)
    
    bsTableEnv.executeSql(
      """
        |insert into print_table
        |select * from hbase_source
        |""".stripMargin)
  }
}