文章 | 链接 |
---|---|
hadoop一主三从安装 | 链接 |
spark on yarn | 链接 |
flink的yarn-session环境 | 链接 |
paimon
目标:
- 1.同步表
- 2.能过 kafka 向 paimon写入
SET parallelism.default =2;
set table.exec.sink.not-null-enforcer = drop;
SET jobmanager.memory.process.size = 2048m;
SET taskmanager.memory.process.size = 10240m;
SET taskmanager.memory.managed.size = 128m;
CREATE CATALOG paimon WITH (
'type' = 'paimon',
'warehouse' = 'hdfs:///data/paimon',
'default-database'='ods'
);
USE CATALOG paimon;
DROP TABLE IF EXISTS xx_REFDES_KAFKA;
CREATE TEMPORARY xx_REFDES_KAFKA
( `PCBID` STRING
,`RID` STRING
,`REFDES` STRING
,`BM\_CIRCUIT\_NO` DECIMAL(38,0)
,`TIMESTAMP` TIMESTAMP
,`PICKUPSTATUS` STRING
,`SERIAL\_NUMBER` STRING
,`FLAG` DECIMAL(2,0)
,`KITID` STRING
,`ID` STRING
,`CREATEDATE` TIMESTAMP
,`ETL` STRING COMMENT 'etl标识'
,`OPT1` STRING COMMENT '备用1'
,`OPT2` STRING COMMENT '备用2'
,`OPT3` STRING COMMENT '备用3'
,`OPT4` STRING COMMENT '备用4'
,`OPT5` STRING COMMENT '备用5'
,`NOZZLEID` STRING COMMENT 'nxt的NOZZLEID'
,`LANENO` STRING COMMENT 'nxt的LANENO'
,`COMPONENTBARCODE` STRING COMMENT 'asm的componentBarcode nxt的Part2DCode'
,`PN` STRING
,`LOTCODE` STRING
,`DATECODE` STRING
,`VERDOR` STRING
,`WORKORDER` STRING
,primary key(ID) not enforced)
WITH(
'connector' = 'kafka',
'topic' = 'TRACE\_LOG\_REFDES',
'properties.bootstrap.servers' = '10.xx.xx.30:9092',
'properties.group.id' = 'xx\_REFDES\_GROUP',
'scan.startup.mode' = 'earliest-offset',
'format' = 'ogg-json'
);
create table if not exists yy_refdes_hive_ro
( `pcbid` string
,`rid` string
,`refdes` string
,`bm\_circuit\_no` decimal(38,0)
,`timestamp` string COMMENT '{"type":"DATE"}'
,`pickupstatus` string
,`serial\_number` string
,`flag` decimal(2,0)
,`kitid` string
,`id` string
,`createdate` string COMMENT '{"type":"DATE"}'
,`etl` string comment 'etl标识'
,`opt1` string comment '备用1'
,`opt2` string comment '备用2'
,`opt3` string comment '备用3'
,`opt4` string comment '备用4'
,`opt5` string comment '备用5'
,`nozzleid` string comment 'nxt的nozzleid'
,`laneno` string comment 'nxt的laneno'
,`componentbarcode` string comment 'asm的componentbarcode nxt的part2dcode'
,`pn` string
,`lotcode` string
,`datecode` string
,`verdor` string
,`workorder` string
,`dt` string
,primary key (id,dt) not enforced)
partitioned by (dt) with (
'connector' = 'paimon',
'file.format' = 'parquet',
'metastore.partitioned-table' = 'true',
'bucket' = '-1',
'partition.expiration-time' = '730 d',
'partition.expiration-check-interval' = '1 d',
![img](https://img-blog.csdnimg.cn/img_convert/c5ec5a743929db04e3b2e8b5d9542db4.png)
![img](https://img-blog.csdnimg.cn/img_convert/7f289337ac49e08db9c497f2c6c1cc53.png)
![img](https://img-blog.csdnimg.cn/img_convert/1301dbf8daff6358e3e0ef16bc551f22.png)
**既有适合小白学习的零基础资料,也有适合3年以上经验的小伙伴深入学习提升的进阶课程,涵盖了95%以上大数据知识点,真正体系化!**
4312805)]
[外链图片转存中...(img-cVsrsRK6-4702074312806)]
**既有适合小白学习的零基础资料,也有适合3年以上经验的小伙伴深入学习提升的进阶课程,涵盖了95%以上大数据知识点,真正体系化!**