EXECUTE CDCSOURCE cdc_demo WITH (
'connector' = 'mysql-cdc',
'hostname' = 'hadoop02',
'port' = '3307',
'username' = 'root',
'password' = '123456',
'scan.startup.mode' = 'initial',
'source.server-time-zone' = 'Asia/Shanghai',
-- 'source.server-time-zone' = 'UTC',
'parallelism' = '1',
'database-name' = 'tpchdb',
'sink.connector' = 'sql-catalog',
-- 'sink.connector' = 'paimon',
'sink.catalog.name' = 'paimon_hive_dinky_catalog',
'sink.catalog.metastore' = 'hive',
'sink.catalog.type' = 'paimon',
'sink.catalog.hive-conf-dir' = '/opt/module/hive/conf',
'sink.catalog.uri' = 'thrift://hadoop02:9083',
-- 'sink.auto-create' = 'true',
'sink.bucket' = '1',
'sink.snapshot.time-retained' = '24h',
'sink.changelog-producer' = 'input',
'sink.catalog.warehouse' = 'hdfs://hadoop03:9000/paimon/test',
'sink.sink.db' = 'tpchdb'
);
我自己的测试实践是目前需要提前建 paimon 表
问题记录:
Cannot find table ‘paimon_hive_dinky_catalog
.tpchdb
.abc
’ in any of t he catalogs [default_catalog, paimon_hive_dinky_catalog], nor as a temporary table.![在这里插入图片描述](https://img-blog.csdnimg.cn/direct/639109eecdf84600a85728761bc53df7.png)
参考:巴别时代基于 Apache Paimon 的 Streaming Lakehouse 的探索与实践
----------------------------------- 分割线 -----------------------------------
添加以下:实现自动建表
自动建表
com.dlink.cdc.sql.catalog.SQLCatalogSinkBuilder.addTableSink
// 切换catalog database
customTableEnvironment.executeSql("use catalog " + catalogName);
customTableEnvironment.executeSql("create database if not exists " + sinkSchemaName);
customTableEnvironment.executeSql("use " + sinkSchemaName);
// 创建目标表
String flinkDDL = FlinkBaseUtil.getFlinkDDL(table, tableName, config, sinkSchemaName, sinkTableName,
pkList);
logger.info(flinkDDL);
customTableEnvironment.executeSql(flinkDDL);
logger.info("Create " + tableName + " FlinkSQL DDL successful...");
com.dlink.utils.FlinkBaseUtil.getSinkConfigurationString
if(StringUtils.contains(configurationString, "sql-catalog")){
StringBuilder sb = new StringBuilder();
String[] optStrSplit = configurationString.split("\n,");
for(String optStr : optStrSplit){
if(StringUtils.contains(optStr,"catalog") || StringUtils.contains(optStr,"connector") ){
}else{
sb.append("," + optStr);
}
}
return sb.deleteCharAt(0).toString();
}else{
return configurationString;
}
问题记录:
cannot contain upper case in hive catalog