Flink-doris-connector sql写入doris on zeppelin优化版

最新推荐文章于 2024-08-13 22:49:19 发布

黄瓜炖啤酒鸭

最新推荐文章于 2024-08-13 22:49:19 发布

阅读量2.1k

点赞数 10

分类专栏： apache doris 文章标签： doris-sql doris-connecter sql插入doris

本文链接：https://blog.csdn.net/qq_31866793/article/details/117362184

版权

本文介绍了如何在Zeppelin中优化Flink-Doris-connector进行SQL写入Doris的操作，包括代码开发、打包细节、Zeppelin执行步骤以及执行效果和总结。重点在于DorisDynamicTableSourceFactory和DorisSinkFunction的实现，解决集群执行时的序列化问题，以及使用阻塞队列确保数据完整写入Doris。在Zeppelin执行时需注意对同一数据源的SQL执行策略，避免任务启动不消费数据的问题。

摘要由CSDN通过智能技术生成

要是觉得写得不错，请务必点个赞，谢谢啊！！！

一，代码开发

1，写入doris的案例代码

2，指定DorisDynamicTableSourceFactory

3，DorisDynamicTableSourceFactory 具体实现

4，DorisSinkFunction具体实现

5，队列类实现

6，doris sink代码，这个比较简单。注意传参就ok了。

一，代码开发

1，写入doris的案例代码

public class DorisSqlTest2 {
    private  static final String DORIS_SQL =  "CREATE TABLE doris_test (\n"+
            " order_number STRING,\n" +
            " order_key STRING,\n" +
            " stock_order STRING,\n" +
            " stock_code STRING,\n" +
            " intermediary STRING,\n" +
            " intermediary_name STRING,\n" +
            " intermediary_accountid STRING,\n" +
            " intermediary_phone STRING,\n" +
            " canal_type STRING\n" +
            " ) WITH (\n"+
            "   'connector' = 'doris',\n"+
            "   'doris.host' = '192.168.6.143',\n"+
            "   'doris.port' = '8030',\n"+
            "   'database-name' = 'example_db',\n"+
            "   'table-name' = 'assure_orders2',\n"+
            "   'username' = 'root',\n"+
            "   'password' = 'root',\n"+
            "   'max.batch' = '500'\n"+
            " )";



    private static final String DATA_GEN =  "CREATE TABLE datagen (\n" +
            " id STRING,\n" +
            " name STRING,\n" +
            " user_age INT,\n" +
            " user_other STRING,\n" +
            " ts AS localtimestamp\n" +
            ") WITH (\n" +
            " 'connector' = 'datagen',\n" +
            " 'rows-per-second'='500',\n" +
            " 'fields.id.length'='7',\n" +
            " 'fields.user_age.min'='1',\n" +
            " 'fields.user_age.max'='100',\n" +
            " 'fields.name.length'='2',\n" +
            " 'fields.user_other.length'='10'\n" +
            ")";



    private  static final String KAFKA_SQL = "CREATE TABLE kafka_test (\n" +
           " order_number STRING,\n" +
            " order_key STRING,\n" +
            " stock_order STRING,\n" +
            " stock_code STRING,\n" +
            " intermediary STRING,\n" +
            " intermediary_name STRING,\n" +
            " intermediary_accountid STRING,\n" +
            " intermediary_phone STRING,\n" +
            " canal_type STRING\n" +
            ") WITH (\n" +
            " 'connector' = 'kafka',\n" +
            " 'topic' = 'dwd.rds_core.plateform_stable.assure_orders',\n" +
            " 'properties.bootstrap.servers' = 'dev-ct6-dc-worker01:9092,dev-ct6-dc-worker02:9092,dev-ct6-dc-worker03:9092',\n" +
            " 'properties.group.id' = 'testGroup',\n" +
            " 'format' = 'json',\n" +
            " 'scan.startup.mode' = 'earliest-offset'\n" +
            ")";

    public static void main(String[] args) {

//        System.out.println("DORIS_SQL = " + DORIS_SQL);
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(3);
        env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime);

        EnvironmentSettings bsSettings = EnvironmentSettings.newInstance().useBlinkPlanner().inStreamingMode().build();
//		StreamTableEnvironment bsTableEnv = StreamTableEnvironment.create(env, bsSettings);
        StreamTableEnvironment bsTableEnv = StreamTableEnvironment.create(env);
        bsTableEnv.executeSql(DORIS_SQL);
        bsTableEnv.executeSql(DATA_GEN);
        bsTableEnv.executeSql(KAFKA_SQL);

        bsTableEnv.executeSql("insert into doris_test select order_number,order_key,stock_order,stock_code,intermediary,intermediary_name,intermediary_accountid,intermediary_phone,canal_type from kafka_test");
//        bsTableEnv.executeSql("insert into kafkaTable select id,name from datagen");
    }
}

2，指定DorisDynamicTableSourceFactory

创建相对应的路径，并且创建类

3，DorisDynamicTableSourceFactory 具体实现

对应的就是sql里面的参数和类型

package org.apache.flink.connector.doris.table;

import org.apache.flink.configuration.ConfigOption;
import org.apache.flink.configuration.ConfigOptions;
import org.apache.flink.configuration.ReadableConfig;
import org.apache.flink.table.api.TableSchema;
import org.apache.flink.table.connector.sink.DynamicTableSink;
import org.apache.flink.table.factories.DynamicTableSinkFactory;
import org.apache.flink.table.factories.FactoryUtil;
import org.apache.flink.table.utils.TableSchemaUtils;

import java.time.Duration;
import java.util.HashSet;
import java.util.Set;

/**
 * @program: flink-neiwang-dev
 * @description: 1
 * @author: Mr.Wang
 * @create: 2020-11-12 14:04
 **/
public class DorisDynamicTableSourceFactory implements DynamicTableSinkFactory {
    //todo 名称叫doris
    public static final String IDENTIFIER = "doris";

    public static final ConfigOption<String> DORIS_HOST = ConfigOptions
            .key("doris.host")
            .stringType()
            .noDefaultValue()
            .withDescription("the doris database url.");


    public static final ConfigOption<String> DORIS_HTTP_PORT = ConfigOptions
            .key("doris.port")
            .stringType()
            .noDefaultValue()
            .withDescription("the doris database url.");


    public static final ConfigOption<String> DATABASE_NAME = ConfigOptions
            .key("database-name")
            .stringType()
            .noDefaultValue()
            .withDescription("the database name.");


    public static final ConfigOption<String> TABLE_NAME = ConfigOptions
            .key("table-name")
            .stringType()
            .noDefaultValue()
            .withDescription("the jdbc table name.");
    public static final ConfigOption<String> USERNAME = ConfigOptions
            .key("username")
            .stringType()
            .noDefaultValue()
            .withDescription("the jdbc user name.");
    public static final ConfigOption<String&g