消费Kafka数据存入HBase（Scala）

最新推荐文章于 2024-01-21 03:38:32 发布

笨鸟先-森

最新推荐文章于 2024-01-21 03:38:32 发布

阅读量319

点赞数 4

文章标签： hbase kafka scala

本文链接：https://blog.csdn.net/qq_70085330/article/details/130751236

版权

该代码示例展示了如何在Scala中使用ApacheFlink从Kafka消费者读取数据，过滤并转换数据，然后将处理后的数据存储到HBase数据库中。主要涉及的功能包括数据过滤、数据转换以及HBase的连接和写入操作。

摘要由CSDN通过智能技术生成

代码如下》》》

package NB

import org.apache.flink.api.common.functions.RichMapFunction
import org.apache.flink.api.common.serialization.SimpleStringSchema
import org.apache.flink.streaming.api.functions.sink.RichSinkFunction
import org.apache.flink.streaming.api.scala._
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer
import org.apache.hadoop.hbase.client.{ConnectionFactory, Put}
import org.apache.hadoop.hbase.util.Bytes
import org.apache.hadoop.hbase.{HBaseConfiguration, TableName}

import java.util.Properties

object text1 {
  def main(args: Array[String]): Unit = {
   val env = StreamExecutionEnvironment.getExecutionEnvironment

   val properties =  new Properties()
   properties.setProperty("bootstrap.servers","192.168.10.102:9092")

   val stream = env.addSource(new FlinkKafkaConsumer[String]("order",new SimpleStringSchema(),properties))

   val dataStream = stream
     .filter(data=>{
       if(data.substring(0,1) == "O"){
         true
       }else{
         false
       }
     })

    val outputStream = dataStream
      .map(data2=>{
        val arr = data2
          .replaceAll("'","")
          .replaceAll("O:","")
          .split(",")
        add(id = arr(0), status = arr(2), price = arr(3), time = arr(4))
      })
      .keyBy(_.id)

    val out= outputStream.map(new RichMapFunction[add,String] {
      override def map(in: add): String = {
        val id = in.id
        val status = in.status
        val price = in.price
        val time = in.time
        //hbase连接配置
        val conf = HBaseConfiguration.create()
        conf.set("hbase.zookeeper.quorum", "master,slave1,slave2")
        conf.set("hbase.zookeeper.property.clientPort", "2181")
        //创建hbase连接
        val connection = ConnectionFactory.createConnection(conf)
        //获取hbase表
        val table = connection.getTable(TableName.valueOf("hainan:WanNing")) //表名

        val put = new Put(Bytes.toBytes(id))//行键
        put.addColumn(
          Bytes.toBytes("name"), //列族
          Bytes.toBytes("id"), //列名
          Bytes.toBytes(id) //数值
        )
        put.addColumn(
          Bytes.toBytes("name"), //列族
          Bytes.toBytes("status"), //列名
          Bytes.toBytes(status) //数值
        )
        put.addColumn(
          Bytes.toBytes("name"), //列族
          Bytes.toBytes("price"), //列名
          Bytes.toBytes(price) //数值
        )
        put.addColumn(
          Bytes.toBytes("name"), //列族
          Bytes.toBytes("time"), //列名
          Bytes.toBytes(time) //数值
        )
        //将数据写入表中
        table.put(put)
        //关闭table
        table.close()
        connection.close()
        id
      }
    })

   out.print("aa")

   env.execute("")

  }
case class add(id:String,status:String,price:String,time:String)
}

验证结果》》》