final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.enableCheckpointing(5000L);
env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
StreamExecutionEnvironment bsEnv = StreamExecutionEnvironment.getExecutionEnvironment();
EnvironmentSettings bsSettings = EnvironmentSettings.newInstance().useBlinkPlanner().inStreamingMode().build();
StreamTableEnvironment tableEnv = StreamTableEnvironment.create(bsEnv, bsSettings);
tableEnv.getConfig().getConfiguration().set(ExecutionCheckpointingOptions.CHECKPOINTING_INTERVAL, Duration.ofSeconds(10));
tableEnv.getConfig().getConfiguration().set(ExecutionCheckpointingOptions.CHECKPOINTING_MODE, CheckpointingMode.EXACTLY_ONCE);
DataStream<TestInfo> gbAlarmInfoStream = addGbAlarmInfoSource(env).filter((FilterFunction<TestInfo>) Objects::nonNull)
.assignTimestampsAndWatermarks(WatermarkStrategy.<TestInfo>forBoundedOutOfOrderness(Duration.ofSeconds(10)).withTimestampAssigner((SerializableTimestampAssigner<TestInfo>) (TestInfo, l) -> TestInfo.getSamplingTime()));
String defaultDatabase = "default";
String name = "vehicle_hive";
String hiveConfDir = "D:\\myprogram\\IdeaProject\\iov-flink-parent\\flink-vehicle-online-stream\\src\\main\\resources";
String version = "2.1.1";
HiveCatalog hive = new HiveCatalog(name, defaultDatabase, hiveConfDir, version);
tableEnv.registerCatalog("vehicle_hive", hive);
tableEnv.useCatalog("vehicle_hive");
tableEnv.getConfig().setSqlDialect(SqlDialect.HIVE);
tableEnv.executeSql("CREATE DATABASE IF NOT EXISTS flink_hive_test");
tableEnv.useDatabase("flink_hive_test");
tableEnv.createTemporaryView("temp_test", gbAlarmInfoStream, $("vin"), $("receiveTimestamp"), $("samplingTime"), $("code"), $("name"), $("level"), $("value"), $("describe"));
tableEnv.executeSql("DROP TABLE IF EXISTS test");
String creatVehicleOnlineTableSql =
"CREATE TABLE test(\n" +
"`vin` String,\n" +
"`receive_timestamp` BIGINT,\n" +
"`sampling_time` BIGINT,\n" +
"`code` String,\n" +
"`name` String,\n" +
"`level` int,\n" +
"`value` String,\n" +
"`describe` String\n" +
") PARTITIONED BY (dt string) STORED AS PARQUET TBLPROPERTIES (\n" +
"'partition.time-extractor.timestamp-pattern'='$dt 00:00:00',\n" +
"'sink.partition-commit.trigger'='partition-time',\n" +
"'sink.partition-commit.delay'='0 h',\n" +
"'sink.partition-commit.policy.kind'='metastore,success-file')";
tableEnv.executeSql(creatVehicleOnlineTableSql);
String insertVehicleOnlineSql = "insert into test SELECT vin, receiveTimestamp as receive_timestamp,samplingTime as sampling_time,code,name,level,`value`,`describe`," +
" from_unixtime(cast(samplingTime/1000 as int),'yyyyMMdd') as dt" +
" FROM temp_test where samplingTime is not null ";
tableEnv.executeSql(insertVehicleOnlineSql);
env.execute();
Flink 写hive
最新推荐文章于 2023-08-29 21:03:00 发布