flink 本地调试连接远程hive-java-scala两个版本的

小皮蛋儿子

已于 2022-06-22 16:05:44 修改

阅读量687

点赞数

分类专栏：大数据文章标签： java flink hive

于 2022-06-22 16:05:29 首次发布

本文链接：https://blog.csdn.net/celltobig/article/details/125411357

版权

大数据专栏收录该内容

16 篇文章 0 订阅

订阅专栏

本文介绍了如何在项目中使用Apache Flink 1.14.0和Blink 1.13.6版本进行表处理，包括依赖配置和Scala API示例。重点展示了如何设置Hive Catalog和执行SQL查询。

摘要由CSDN通过智能技术生成

flink 1.14.0

planner 1.14.0

blink 1.13.6

pom.xml

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">

    <modelVersion>4.0.0</modelVersion>

    <groupId>com.cell</groupId>
    <artifactId>FlinktTest</artifactId>
    <version>1.0-SNAPSHOT</version>
    <build>
        <plugins>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-compiler-plugin</artifactId>
                <configuration>
                    <source>1.8</source>
                    <target>1.8</target>
                </configuration>
            </plugin>
        </plugins>
    </build>

    <properties>
        <blink.flink.version>1.13.6</blink.flink.version>
        <flink.version>1.14.0</flink.version>
        <scala.binary.version>2.12</scala.binary.version>
    </properties>

    <dependencies>
        <!-- Flink modules  start -->

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-scala_${scala.binary.version}</artifactId>
            <version>${flink.version}</version>
            <!--<scope>provided</scope>-->
            <exclusions>
                <exclusion>
                    <artifactId>scala-library</artifactId>
                    <groupId>org.scala-lang</groupId>
                </exclusion>
            </exclusions>
        </dependency>

        <!--<dependency>-->
        <!--<groupId>org.apache.flink</groupId>-->
        <!--<artifactId>flink-avro</artifactId>-->
        <!--<version>${flink.version}</version>-->
        <!--<exclusions>-->
        <!--<exclusion>-->
        <!--<artifactId>slf4j-api</artifactId>-->
        <!--<groupId>org.slf4j</groupId>-->
        <!--</exclusion>-->
        <!--<exclusion>-->
        <!--<artifactId>commons-compress</artifactId>-->
        <!--<groupId>org.apache.commons</groupId>-->
        <!--</exclusion>-->
        <!--</exclusions>-->
        <!--</dependency>-->

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-streaming-scala_${scala.binary.version}</artifactId>
            <version>${flink.version}</version>
            <!--<scope>provided</scope>-->
        </dependency>

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-table-common</artifactId>
            <version>${flink.version}</version>
            <!--<scope>provided</scope>-->
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-table-api-scala_${scala.binary.version}</artifactId>
            <version>${flink.version}</version>
            <!--<scope>provided</scope>-->
        </dependency>

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-table-api-java-bridge_${scala.binary.version}</artifactId>
            <version>${flink.version}</version>
            <!--<scope>provided</scope>-->
        </dependency>

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-table-api-scala-bridge_${scala.binary.version}</artifactId>
            <version>${flink.version}</version>
            <!--<scope>provided</scope>-->
        </dependency>

        <!--<dependency>-->
        <!--<groupId>org.apache.flink</groupId>-->
        <!--<artifactId>flink-table-planner-blink_${scala.binary.version}</artifactId>-->
        <!--<version>${blink.flink.version}</version>-->
        <!--&lt;!&ndash;<scope>provided</scope>&ndash;&gt;-->
        <!--<exclusions>-->
        <!--<exclusion>-->
        <!--<artifactId>slf4j-api</artifactId>-->
        <!--<groupId>org.slf4j</groupId>-->
        <!--</exclusion>-->
        <!--</exclusions>-->
        <!--</dependency>-->

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-table-planner_${scala.binary.version}</artifactId>
            <version>${flink.version}</version>
            <!--<scope>provided</scope>-->
        </dependency>


        <!--<dependency>-->
        <!--<groupId>org.apache.flink</groupId>-->
        <!--<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>-->
        <!--<version>${flink.version}</version>-->
        <!--&lt;!&ndash;<scope>provided</scope>&ndash;&gt;-->
        <!--<exclusions>-->
        <!--<exclusion>-->
        <!--<artifactId>commons-cli</artifactId>-->
        <!--<groupId>commons-cli</groupId>-->
        <!--</exclusion>-->
        <!--<exclusion>-->
        <!--<artifactId>slf4j-api</artifactId>-->
        <!--<groupId>org.slf4j</groupId>-->
        <!--</exclusion>-->
        <!--<exclusion>-->
        <!--<artifactId>scala-parser-combinators_2.12</artifactId>-->
        <!--<groupId>org.scala-lang.modules</groupId>-->
        <!--</exclusion>-->
        <!--</exclusions>-->
        <!--</dependency>-->

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-clients_${scala.binary.version}</artifactId>
            <version>${flink.version}</version>
            <!--<scope>provided</scope>-->
            <exclusions>
                <exclusion>
                    <groupId>org.slf4j</groupId>
                    <artifactId>slf4j-log4j12</artifactId>
                </exclusion>
                <exclusion>
                    <artifactId>slf4j-api</artifactId>
                    <groupId>org.slf4j</groupId>
                </exclusion>
                <exclusion>
                    <groupId>log4j</groupId>
                    <artifactId>log4j</artifactId>
                </exclusion>
            </exclusions>
        </dependency>

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-json</artifactId>
            <version>${flink.version}</version>
            <!--<scope>provided</scope>-->
        </dependency>


        <!-- rocksdb-->
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-statebackend-rocksdb_${scala.binary.version}</artifactId>
            <version>${flink.version}</version>
            <!--<scope>provided</scope>-->
        </dependency>

        <!-- Hive Connector的支持，仅在编译时生效-->
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-connector-hive_${scala.binary.version}</artifactId>
            <version>${flink.version}</version>
            <!--<scope>provided</scope>-->
        </dependency>

        <dependency>
            <groupId>org.apache.hive</groupId>
            <artifactId>hive-exec</artifactId>
            <version>2.1.1</version>
            <!--<scope>provided</scope>-->
            <exclusions>
                <exclusion>
                    <groupId>org.apache.calcite</groupId>
                    <artifactId>calcite-core</artifactId>
                </exclusion>
                <exclusion>
                    <groupId>org.apache.calcite</groupId>
                    <artifactId>calcite-avatica</artifactId>
                </exclusion>
                <exclusion>
                    <artifactId>hadoop-hdfs</artifactId>
                    <groupId>org.apache.hadoop</groupId>
                </exclusion>
            </exclusions>
        </dependency>


        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-json</artifactId>
            <version>1.14.0</version>
        </dependency>
        <!-- Flink modules  end -->


        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-mapreduce-client-core</artifactId>
            <version>2.7.2</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-common</artifactId>
            <version>2.7.2</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-mapreduce-client-common</artifactId>
            <version>2.7.2</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
            <version>2.7.2</version>
        </dependency>


    </dependencies>


</project>

java

package com.cell.flink.java;

import org.apache.flink.table.api.EnvironmentSettings;
import org.apache.flink.table.api.SqlDialect;
import org.apache.flink.table.api.TableEnvironment;
import org.apache.flink.table.catalog.hive.HiveCatalog;

public class Demo {


    public static void main(String[] args) {


        EnvironmentSettings settings = EnvironmentSettings.newInstance().inStreamingMode().build();

        TableEnvironment tableEnv = TableEnvironment.create(settings);

        String hiveConfDir = Thread.currentThread().getContextClassLoader().getResource("").getPath();

        System.out.println("hiveConfDir=" + hiveConfDir);

        tableEnv.getConfig().setSqlDialect(SqlDialect.HIVE);
        HiveCatalog hive = new HiveCatalog("my_hive","test002",hiveConfDir);
        tableEnv.registerCatalog("my_hive",hive);


        tableEnv.useCatalog("my_hive");
        tableEnv.useDatabase("test002");

        tableEnv.executeSql("show tables").print();



    }
}

scala

package com.cell.flink

import org.apache.flink.streaming.api.scala._
import org.apache.flink.table.api.bridge.scala.StreamTableEnvironment
import org.apache.flink.table.api.{EnvironmentSettings, TableEnvironment}
import org.apache.flink.table.catalog.hive.HiveCatalog
import org.apache.flink.table.module.hive.HiveModule


object Test {


  def main(args: Array[String]): Unit = {

    val env = StreamExecutionEnvironment.getExecutionEnvironment
    env.setParallelism(1)
    val tableEnv = StreamTableEnvironment.create(env)

//    val settings = EnvironmentSettings.newInstance()
//      .inStreamingMode()
//      .build()
//    val settings = EnvironmentSettings.inStreamingMode()
//    val tableEnv = TableEnvironment.create(settings)

    val hiveConfDir = Thread.currentThread().getContextClassLoader().getResource("").getPath()
//    println(hiveConfDir)

    val myHive = new HiveCatalog("my_hive","test002",hiveConfDir)
    tableEnv.registerCatalog("my_hive",myHive)
    tableEnv.useCatalog("my_hive")
    tableEnv.useDatabase("test002")
    tableEnv.executeSql(
      """
        |show tables
      """.stripMargin).print()

    tableEnv.listTables()











  }

}

效果