Flink 写hive

 final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.enableCheckpointing(5000L);
        env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
        StreamExecutionEnvironment bsEnv = StreamExecutionEnvironment.getExecutionEnvironment();
        EnvironmentSettings bsSettings = EnvironmentSettings.newInstance().useBlinkPlanner().inStreamingMode().build();
        StreamTableEnvironment tableEnv = StreamTableEnvironment.create(bsEnv, bsSettings);
        tableEnv.getConfig().getConfiguration().set(ExecutionCheckpointingOptions.CHECKPOINTING_INTERVAL, Duration.ofSeconds(10));
        tableEnv.getConfig().getConfiguration().set(ExecutionCheckpointingOptions.CHECKPOINTING_MODE, CheckpointingMode.EXACTLY_ONCE);
        DataStream<TestInfo> gbAlarmInfoStream = addGbAlarmInfoSource(env).filter((FilterFunction<TestInfo>) Objects::nonNull)
                .assignTimestampsAndWatermarks(WatermarkStrategy.<TestInfo>forBoundedOutOfOrderness(Duration.ofSeconds(10)).withTimestampAssigner((SerializableTimestampAssigner<TestInfo>) (TestInfo, l) -> TestInfo.getSamplingTime()));
        String defaultDatabase = "default";
        String name = "vehicle_hive";
        String hiveConfDir = "D:\\myprogram\\IdeaProject\\iov-flink-parent\\flink-vehicle-online-stream\\src\\main\\resources";
        String version = "2.1.1";
        HiveCatalog hive = new HiveCatalog(name, defaultDatabase, hiveConfDir, version);
        tableEnv.registerCatalog("vehicle_hive", hive);
        tableEnv.useCatalog("vehicle_hive");
        tableEnv.getConfig().setSqlDialect(SqlDialect.HIVE);
        tableEnv.executeSql("CREATE DATABASE IF NOT EXISTS flink_hive_test");
        tableEnv.useDatabase("flink_hive_test");
        tableEnv.createTemporaryView("temp_test", gbAlarmInfoStream, $("vin"), $("receiveTimestamp"), $("samplingTime"), $("code"), $("name"), $("level"), $("value"), $("describe"));
        tableEnv.executeSql("DROP TABLE IF EXISTS test");
        String creatVehicleOnlineTableSql =
                "CREATE TABLE test(\n" +
                        "`vin` String,\n" +
                        "`receive_timestamp` BIGINT,\n" +
                        "`sampling_time` BIGINT,\n" +
                        "`code` String,\n" +
                        "`name` String,\n" +
                        "`level` int,\n" +
                        "`value` String,\n" +
                        "`describe` String\n" +
                        ") PARTITIONED BY (dt string) STORED AS PARQUET TBLPROPERTIES (\n" +
                        "'partition.time-extractor.timestamp-pattern'='$dt 00:00:00',\n" +
                        "'sink.partition-commit.trigger'='partition-time',\n" +
                        "'sink.partition-commit.delay'='0 h',\n" +
                        "'sink.partition-commit.policy.kind'='metastore,success-file')";

        tableEnv.executeSql(creatVehicleOnlineTableSql);
        String insertVehicleOnlineSql = "insert into test SELECT vin, receiveTimestamp as receive_timestamp,samplingTime as sampling_time,code,name,level,`value`,`describe`," +
                " from_unixtime(cast(samplingTime/1000 as int),'yyyyMMdd') as dt" +
                " FROM temp_test where samplingTime is not null ";

        tableEnv.executeSql(insertVehicleOnlineSql);
        env.execute();
  • 1
    点赞
  • 1
    收藏
    觉得还不错? 一键收藏
  • 1
    评论
评论 1
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值