背景:在使用spark Structured Streaming将数据写入到kafka时,hudi compaction未按照相关参数正常执行。
无法正常compaction代码如下:
val spark = SparkSession
.builder
.config("spark.serializer","org.apache.spark.serializer.KryoSerializer")
.appName("test_leo")
.enableHiveSupport()
.getOrCreate()
val df =
spark.readStream
.format("kafka")
.option("kafka.bootstrap.servers", "ip:port")
.option("subscribe", "hudi_leo_test").option("startingOffsets", "earliest").option("maxOffsetsPerTrigger", 2).option("failOnDataLoss",true)
.load()
df.writeStream.option("checkpointLocation","/spark/checkpoints/teststreaming01")
.format("hudi")
.option(DataSourceWriteOptions.OPERATION.key(), DataSourceWriteOptions.UPSERT_OPERATION_OPT_VAL)
.option(DataSourceWriteOptions.PAYLOAD_CLASS_NAME.key(), "org.apache.hudi.common.model.OverwriteWithLatestAvroPayload")
.option(DataSourceWriteOptions.STREAMING_IGNORE_FAILED_BATCH.key(), "false")
.option(DataSourceWriteOptions.TABLE_TYPE.key(), "MERGE_ON_READ")
.option(HoodieWriteConfig.TBL_NAME.key(), "test_hudi_leo_com")
.option(DataSourceWriteOptions.RECORDKEY_FIELD.key(), "id")
.option(DataSourceWriteOptions.PRECOMBINE_FIELD.key(), "ts")
.option(DataSourceWriteOptions.HIVE_SYNC_ENABLED.key(), true)
.option(DataSourceWriteOptions.HIVE_DATABASE.key(), "temp_db")
.option(DataSourceWriteOptions.HIVE_TABLE.key(), "test_hudi_leo_com")
.option(DataSourceWriteOptions.HIVE_URL.key(), hive_url)
.option(DataSourceWriteOptions.HIVE_USER.key(), "hudi")
.option(DataSourceWriteOptions.HIVE_PASS.key(), "hudi")
.option(HoodieWriteConfig.INSERT_PARALLELISM_VALUE.key,"40")
.option(HoodieWriteConfig.BULKINSERT_PARALLELISM_VALUE.key,"40")
.option(HoodieWriteConfig.UPSERT_PARALLELISM_VALUE.key,"40")
.option(HoodieWriteConfig.DELETE_PARALLELISM_VALUE.key,"40")
.option(HoodieIndexConfig.INDEX