CC00015.bdpositions——|Hadoop&实时数仓.V15|——|项目.v15|ODS层处理|将kafka中的维表写入DIM层.V3|

一、编程实现程序：将Kafka中的维度表写入DIM层

### --- 编程实现：KafkaToHBase：将Kafka作为Source，Flink作为消费者从Kafka中获取数据

package ods

import java.util

import com.alibaba.fastjson.{JSON, JSONObject}
import org.apache.flink.streaming.api.scala.{DataStream, StreamExecutionEnvironment}
import org.apache.flink.api.scala._

/**
 * 1.从kafka的canal这个topic获取数据-----FlinkKafkaConsumer
 * 2.把获取到的json格式的数据进行格式转化-----fastjson
 * type,database, table,data(jsonArray)
 * 3.把转化好的数据保存到HBase中
 */
object KafkaToHBase {

  def main(args: Array[String]): Unit = {
    val env = StreamExecutionEnvironment.getExecutionEnvironment
    val kafkaConsumer = new SourceKafka().getKafkaSource("canal")
    kafkaConsumer.setStartFromLatest()

    val sourceStream = env.addSource(kafkaConsumer)
    // sourceStream.print()
    // type,database, table,data(jsonArray)
    //
    val mapped: DataStream[util.ArrayList[TableObject]] = sourceStream.map(x => {
      val jsonObj: JSONObject = JSON.parseObject(x)
      val database: AnyRef = jsonObj.get("database")
      val table: AnyRef = jsonObj.get("table")
      val typeInfo: AnyRef = jsonObj.get("type")

      val objects = new util.ArrayList[TableObject]()
      jsonObj.getJSONArray("data").forEach(x => {
//        print(database.toString + ".." + table.toString + "..." + typeInfo.toString + ".." + x.toString)
        objects.add(TableObject(database.toString, table.toString, typeInfo.toString, x.toString))
      })
      objects
    })


    /**
     * 将数据下沉到HBase中保存
     * 1.拿到当前的数据
     * 2、addSink()--- 自定义下沉器SinkHBase
     */
    mapped.addSink(new SinkHBase)

    env.execute()
  }
}

### --- 编程实现：SinkHBase：从Kafka中获取表数据：将数据下沉到hbash中

package ods

import java.util

import com.alibaba.fastjson.JSON
import org.apache.flink.configuration
import org.apache.flink.streaming.api.functions.sink.{RichSinkFunction, SinkFunction}
import org.apache.hadoop.hbase.TableName
import org.apache.hadoop.hbase.client.{Connection, Delete, Put, Table}

class SinkHBase extends RichSinkFunction[util.ArrayList[TableObject]]{
  var connection : Connection = _
  var hbTable : Table = _
  /**
   * 实例化 HBase
   * connection
   * hbTable
   * @param parameters
   */
  override def open(parameters: configuration.Configuration): Unit = {
    connection = new ConnHBase().connToHbase
    // hbTable = connection.getTable(TableName.valueOf("yanqi_area"))
    hbTable = connection.getTable(TableName.valueOf("yanqi_trade_orders"))

  }

  override def close(): Unit = {
    if(hbTable != null) {
      hbTable.close()
    }
    if (connection != null) {
      connection.close()
    }
  }

  /**
   * 每来一条数据，会执行一次
   * @param value
   * @param context
   */
  override def invoke(value: util.ArrayList[TableObject], context: SinkFunction.Context[_]): Unit = {
    value.forEach(x => {
      println(x.toString)
      val database: String = x.database
      val tableName: String = x.tableName
      val typeInfo: String = x.typeInfo

      if(database.equalsIgnoreCase("dwshow") && tableName.equalsIgnoreCase("yanqi_trade_orders")) {
        if(typeInfo.equalsIgnoreCase("insert")) {
          value.forEach(x => {
            val info: DataInfo = JSON.parseObject(x.dataInfo, classOf[DataInfo])
            insertTradeOrders(hbTable,info)
          })
        } else if(typeInfo.equalsIgnoreCase("update")) {

        } else if (typeInfo.equalsIgnoreCase("delete")) {

        }
      }

      if(database.equalsIgnoreCase("dwshow") && tableName.equalsIgnoreCase("yanqi_area")) {
        if(typeInfo.equalsIgnoreCase("insert")) {
          value.forEach(x => {
            val info: AreaInfo = JSON.parseObject(x.dataInfo, classOf[AreaInfo])
            insertArea(hbTable,info)
          })
        } else if(typeInfo.equalsIgnoreCase("update")) {
          value.forEach(x => {
            val info: AreaInfo = JSON.parseObject(x.dataInfo, classOf[AreaInfo])
            insertArea(hbTable,info)
          })
        } else if (typeInfo.equalsIgnoreCase("delete")) {
          value.forEach(x => {
            val info: AreaInfo = JSON.parseObject(x.dataInfo, classOf[AreaInfo])
            deleteArea(hbTable,info)
          })

        }
      }

      })
  }

  //yanqi_area省份城市区域表，根据id删除数据
  def deleteArea(hbTable: Table, areaInfo: AreaInfo): Unit = {
    val delete = new Delete(areaInfo.id.getBytes)
    hbTable.delete(delete)
  }

  def insertArea(hbTable: Table, areaInfo: AreaInfo) : Unit = {
    println(areaInfo.toString)
    val put = new Put(areaInfo.id.getBytes())
    put.addColumn("f1".getBytes(), "name".getBytes(), areaInfo.name.getBytes())
    put.addColumn("f1".getBytes(), "pid".getBytes(), areaInfo.pid.getBytes())
    put.addColumn("f1".getBytes(), "sname".getBytes(), areaInfo.sname.getBytes())
    put.addColumn("f1".getBytes(), "level".getBytes(), areaInfo.level.getBytes())
    put.addColumn("f1".getBytes(), "citycode".getBytes(), areaInfo.citycode.getBytes())
    put.addColumn("f1".getBytes(), "yzcode".getBytes(), areaInfo.yzcode.getBytes())
    put.addColumn("f1".getBytes(), "mername".getBytes(), areaInfo.mername.getBytes())
    put.addColumn("f1".getBytes(), "lng".getBytes(), areaInfo.Lng.getBytes())
    put.addColumn("f1".getBytes(), "lat".getBytes(), areaInfo.Lat.getBytes())
    put.addColumn("f1".getBytes(), "pinyin".getBytes(), areaInfo.pinyin.getBytes())

    hbTable.put(put)
  }
  def insertTradeOrders(hbTable: Table, dataInfo: DataInfo): Unit = {
    val put = new Put(dataInfo.orderId.getBytes)
    put.addColumn("f1".getBytes,"modifiedTime".getBytes,dataInfo.modifiedTime.getBytes())
    put.addColumn("f1".getBytes,"orderNo".getBytes,dataInfo.orderNo.getBytes())
    put.addColumn("f1".getBytes,"isPay".getBytes,dataInfo.isPay.getBytes())
    put.addColumn("f1".getBytes,"orderId".getBytes,dataInfo.orderId.getBytes())
    put.addColumn("f1".getBytes,"tradeSrc".getBytes,dataInfo.tradeSrc.getBytes())
    put.addColumn("f1".getBytes,"payTime".getBytes,dataInfo.payTime.getBytes())
    put.addColumn("f1".getBytes,"productMoney".getBytes,dataInfo.productMoney.getBytes())
    put.addColumn("f1".getBytes,"totalMoney".getBytes,dataInfo.totalMoney.getBytes())
    put.addColumn("f1".getBytes,"dataFlag".getBytes,dataInfo.dataFlag.getBytes())
    put.addColumn("f1".getBytes,"userId".getBytes,dataInfo.userId.getBytes())
    put.addColumn("f1".getBytes,"areaId".getBytes,dataInfo.areaId.getBytes())
    put.addColumn("f1".getBytes,"createTime".getBytes,dataInfo.createTime.getBytes())
    put.addColumn("f1".getBytes,"payMethod".getBytes,dataInfo.payMethod.getBytes())
    put.addColumn("f1".getBytes,"isRefund".getBytes,dataInfo.isRefund.getBytes())
    put.addColumn("f1".getBytes,"tradeType".getBytes,dataInfo.tradeType.getBytes())
    put.addColumn("f1".getBytes,"status".getBytes,dataInfo.status.getBytes())

    hbTable.put(put)
  }
}