切换spark版本,本例子使用 spark2.4.5
source change_spark_version spark-2.4.5.0
spark-shell //进入 scala>
创建表
import org.apache.iceberg.catalog.Tabicedentifier
import org.apache.iceberg.spark.SparkSchemaUtil
import org.apache.iceberg.hive.HiveCatalog;
import org.apache.spark.SparkConf;
import org.apache.iceberg.catalog.TableIdentifier;
import org.apache.spark.sql.connector.catalog.TableProvider;
val catalog = new HiveCatalog(spark.sparkContext.hadoopConfiguration)
val data = Seq((1, "a"), (2, "b"), (3, "c")).toDF("id", "data") // 数据
val schema = SparkSchemaUtil.convert(data.schema)
// 数据库名称,iceberg表名称
val name = TableIdentifier.of("testdb", "ice_table3")
val table = catalog.createTable(name, schema)
// 追加数据到iceberg表中
data.write.format("iceberg").mode("append").save("testdb.ice_table3")
// 加载iceberg表
spark.read.format("iceberg").load("testdb.ice_table3")
查询表:
// 利用视图来访问iceberg表中的内容
val df = spark.read.format("iceberg").load("testdb.ice_table3")
df.createOrReplaceTempView("view")
spark.sql("""SELECT * FROM view""").show()
删除表:
spark.sql("DROP TABLE ice_table1").show()