Flink 1.12.1 Kafka连接 Source 和 Sink

最新推荐文章于 2024-07-15 17:05:03 发布

土豆土豆土豆

最新推荐文章于 2024-07-15 17:05:03 发布

阅读量1k

点赞数

分类专栏： Flink 文章标签： flink kafka

本文链接：https://blog.csdn.net/weixin_42066446/article/details/113242891

版权

Flink 专栏收录该内容

4 篇文章 0 订阅

订阅专栏

Flink 1.12.1 Kafka Source Sink

Flink 1.12.1 Kafka 连接 Soure 和 Sink

参考Flink官网地址

scala版本：2.12
Flink版本：1.12.1
kafka版本：2.7.0

添加依赖

<dependency>
    <groupId>org.apache.flink</groupId>
    <artifactId>flink-connector-kafka_2.11</artifactId>
    <version>1.12.1</version>
</dependency>

Source 代码

package com.lhn.source

import java.util.Properties

import org.apache.flink.api.common.serialization.SimpleStringSchema
import org.apache.flink.streaming.api.scala._
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer

object SourceTest {

  def main(args: Array[String]): Unit = {

    val env = StreamExecutionEnvironment.getExecutionEnvironment

    val property = new Properties()

    property.setProperty("bootstrap.servers","localhost:9092")
    property.setProperty("group.id","consumer-group")
    property.setProperty("key.deserializer","org.apache.kafka.common.serialization.StringDeserializer")
    property.setProperty("value.deserializer","org.apache.kafka.common.serialization.StringDeserializer")
    property.setProperty("auto.offset.reset", "latest")


    val dStream: DataStream[String] = env.addSource(new FlinkKafkaConsumer[String]("sensor", new SimpleStringSchema(), property))

    dStream.print()

    env.execute("Test Kafka Source")

  }

}

Sink代码

package com.lhn.sink

import java.util.Properties

import com.lhn.bean.SensorSource
import com.lhn.test.CompanyType
import org.apache.flink.api.common.serialization.SimpleStringSchema
import org.apache.flink.streaming.api.functions.source.SourceFunction
import org.apache.flink.streaming.api.scala._
import org.apache.flink.streaming.connectors.kafka.{FlinkKafkaConsumer, FlinkKafkaProducer}
import org.apache.kafka.clients.consumer.KafkaConsumer


object SinkKafkaTest {

  def main(args: Array[String]): Unit = {
    val env: StreamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment
    env.setParallelism(1)

    val sensorDataStream: DataStream[SensorReading] = env.addSource(new SensorSource())
    val value: DataStream[String] = sensorDataStream
      .map(t => t.sensor + "," + t.curTemp.toString + "," + t.curTime.toString)
    value.addSink(new FlinkKafkaProducer[String]("localhost:9092","sink",new SimpleStringSchema()))

    env.execute("Kafka Sink Test")
  }
  case class SensorReading(sensor: String, curTime: Long, curTemp: Double)
}