flink 读取kafka数据写入到mysql scala版

package _20210628

import java.sql.{Connection, DriverManager, PreparedStatement}
import java.util.Properties


import org.apache.flink.api.common.serialization.{DeserializationSchema, SimpleStringSchema}
import org.apache.flink.configuration.Configuration
import org.apache.flink.streaming.api.functions.sink.{RichSinkFunction, SinkFunction}
import org.apache.flink.streaming.api.scala.{DataStream, StreamExecutionEnvironment}
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer011
import org.apache.kafka.clients.consumer.ConsumerConfig
import org.apache.flink.streaming.api.scala._

import scala.util.Properties

object Sink_Mysql {
  def main(args: Array[String]): Unit = {


    val env: StreamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment

    env.setParallelism(1)

    val prop = new Properties()
    prop.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "192.168.107.103:9092")
    prop.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, " org.apache.kafka.common.serialization.StringDeserializer")
    prop.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, " org.apache.kafka.common.serialization.StringDeserializer")
    prop.put(ConsumerConfig.GROUP_ID_CONFIG, "scala_sink_mysql_group1")


    val flinkKafkaConsumer011 = new FlinkKafkaConsumer011[String]("sensor", new SimpleStringSchema(), prop)
    val kafkaStream: DataStream[String] = env.addSource(flinkKafkaConsumer011)

    val value: DataStream[sensor] = kafkaStream.map(x => {
      val strs: Array[String] = x.split(",")
      val sensor = new sensor(strs(0), strs(1).toLong, strs(2).toDouble)
      sensor
    })
    value.print()

    value.addSink(new MysqlSink())


    env.execute("scala_sinkMysql")


  }

}

case class sensor(id: String, timestamp: Long, temp: Double)

class MysqlSink extends RichSinkFunction[sensor] {
  var connection: Connection = null

  override def open(parameters: Configuration): Unit = {

    connection = DriverManager.getConnection(
      "jdbc:mysql://192.168.107.103:3306/flinkDemo",
      "root",
      "ok"
    )


  }
  override def invoke(value: sensor, context: SinkFunction.Context[_]): Unit = {
    val insert: PreparedStatement = connection.prepareStatement("insert into sensor_temp(id,temp) values(?,?)")
    val update: PreparedStatement = connection.prepareStatement("update sensor_temp set temp=? where id=?")
    update.setDouble(1,value.temp)
    update.setString(2,value.id)
    update.execute()

    if (update.getUpdateCount==0){
      insert.setString(1,value.id)
      insert.setDouble(2,value.temp)
      insert.execute()
    }

  }


  override def close(): Unit = connection.close()
}

在这里插入图片描述
在这里插入图片描述

说明:
java版和scala版 基本一样 java没有样例类 单独编写一个sensor类就可以

  • 1
    点赞
  • 6
    收藏
    觉得还不错? 一键收藏
  • 3
    评论

“相关推荐”对你有帮助么?

  • 非常没帮助
  • 没帮助
  • 一般
  • 有帮助
  • 非常有帮助
提交
评论 3
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值