import org.apache.spark.sql.{DataFrame, SparkSession}
object SparkSQLHBaseSource {
def main(args: Array[String]): Unit = {
val sparkSession: SparkSession = SparkSession.builder().master("local[1]").appName("sparkSQLHBase").getOrCreate()
sparkSession.sparkContext.setLogLevel("WARN")
val order: DataFrame = sparkSession.read
.format("com.travel.programApp.hbaseSource.HBaseSource")
.options(Map(
GlobalConfigUtils.getProp("sparksql_table_schema") -> GlobalConfigUtils.getProp("order.sparksql_table_schema"),
GlobalConfigUtils.getProp("hbase_table_name") -> GlobalConfigUtils.getProp("syn.table.order_info"),
GlobalConfigUtils.getProp("hbase_table_schema") -> GlobalConfigUtils.getProp("order.hbase_table_schema")
)).load()
val user: DataFrame = sparkSession.read
.format("com.travel.programApp.hbaseSource.HBaseSource")
.options(Map(
GlobalConfigUtils.getProp("sparksql_table_schema") -> GlobalConfigUtils.getProp("drivers.spark_sql_table_schema"),
GlobalConfigUtils.getProp("hbase_table_name") -> GlobalConfigUtils.getProp("syn.table.driver_info"),
GlobalConfigUtils.getProp("hbase_table_schema") -> GlobalConfigUtils.getProp("driver.hbase_table_schema")
)).load()
val renter: DataFrame = sparkSession.read
.format("com.travel.programApp.hbaseSource.HBaseSource")
.options(Map(
GlobalConfigUtils.getProp("sparksql_table_schema") -> GlobalConfigUtils.getProp("registe.sparksql_table_schema"),
GlobalConfigUtils.getProp("hbase_table_name") -> GlobalConfigUtils.getProp("syn.table.renter_info"),
GlobalConfigUtils.getProp("hbase_table_schema") -> GlobalConfigUtils.getProp("registe.hbase_table_schema")
)).load()
//注册
order.createOrReplaceTempView("order")
driver.createOrReplaceTempView("user")
renter.createOrReplaceTempView("renter")
//cache
sparkSession.sqlContext.cacheTable("order")
sparkSession.sqlContext.cacheTable("user")
sparkSession.sqlContext.cacheTable("renter")
// OrderTransation.init(sparkSession)
// RenterTransation.init(sparkSession)
// DriverTransation.init(sparkSession)
// HotOrderTransation.init(sparkSession)
// HotAreaOrder.init(sparkSession)
}
}
SparkSQL整合多个hbase源
最新推荐文章于 2023-05-04 23:00:00 发布