代码如下所示:
public class KafkaToHive {
public static void main(String[] args) {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
//1.12以后默认都是EventTime,这是过期方法,并且不在提IngestionTime
//env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
//如果要使用ProcessingTime,可以关闭watermark
env.getConfig().setAutoWatermarkInterval(0);
env.setParallelism(2);
EnvironmentSettings settings = EnvironmentSettings.newInstance()
.useBlinkPlanner()
.inStreamingMode()
.build();
StreamTableEnvironment tEnv = StreamTableEnvironment.create(env, settings);
//设置为exactly-once
tEnv.getConfig().getConfiguration().set(ExecutionCheckpointingOptions.CHECKPOINTING_MODE, CheckpointingMode.EXACTLY_ONCE);
tEnv.getConfig().getConfiguration().set(ExecutionCheckpointingOptions.CHECKPOINTING_INTERVAL, Duration.ofSeconds(20));
//配置hive
String catalogName = "my_catalog";
String db = "default";
String hiveConfPath = "./src/main/resources";
HiveCatalog hiveCatalog = new HiveCatalog(catalogName, db, hiveConfPath);
//注册并使用
tEnv.registerCatalog(catalogName,hiveCatalog);
tEnv.useCatalog(catalogName);
tEnv.executeSql("CREATE DATABASE IF NOT EXISTS stream");
tEnv.executeSql("DROP TABLE IF EXISTS stream.kafka_log");
//创建kafka的表
tEnv.executeSql("create table stream.kafka_log(\n" +
"user_id String,\n" +
"order_amount Double,\n" +
"order_amount Double,\n" +
"log_ts Timestamp(3),\n" +
"WATERMARK FOR log_ts AS log_ts -INTERVAL '5' SECOND\n" +
" )WITH(\n" +
" 'connector' = 'kafka',\n" +
" 'topic' = 'test',\n" +
" 'properties.bootstrap.servers' = 'node1:9092',\n" +
" 'properties.group.id' = 'flink1',\n" +
" 'scan.startup.mode' = 'earliest-offset',\n" +
" 'format' = 'json',\n" +
" 'json.fail-on-missing-field' = 'false\n" +
" 'json.ignore-parse-errors' = 'true'\n" +
" )");
//开始在hive中创建表
tEnv.getConfig().setSqlDialect(SqlDialect.HIVE);
tEnv.executeSql("CREATE DATABASE IF NOT EXISTS test");
tEnv.executeSql("DROP TABLE IF EXISTS test.hive_log");
tEnv.executeSql(" create table test.hive_log(\n" +
" user_id String,\n" +
" order_amount double,\n" +
" )partition by (\n" +
" dt STRING,\n"