scala 实现flink cdc

导入相关pom 文件 <properties> <scala.version>2.11.0</scala.version> <flink.version>1.12.3</flink.version> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <maven.c
摘要由CSDN通过智能技术生成

导入相关pom 文件

  <properties>
        <scala.version>2.11.0</scala.version>
        <flink.version>1.12.3</flink.version>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
        <maven.compiler.source>1.8</maven.compiler.source>
        <maven.compiler.target>1.8</maven.compiler.target>
    </properties>

    <dependencies>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-scala_2.11</artifactId>
            <version>1.11.3</version>
        </dependency>
      
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-streaming-scala_2.11</artifactId>
            <version>${flink.version}</version>
        </dependency>

        <dependency>
            <groupId>org.apache.flink
  • 0
    点赞
  • 5
    收藏
    觉得还不错? 一键收藏
  • 1
    评论
以下是使用 Flink 实现 MySQL CDCScala 代码示例: ```scala import org.apache.flink.streaming.api.scala._ import org.apache.flink.streaming.api.functions.source.SourceFunction import org.apache.flink.streaming.api.functions.AssignerWithPunctuatedWatermarks import org.apache.flink.streaming.api.watermark.Watermark import org.apache.flink.streaming.api.functions.sink.SinkFunction import org.apache.flink.streaming.api.functions.ProcessFunction import org.apache.flink.util.Collector import org.apache.flink.streaming.api.TimeCharacteristic import org.apache.flink.streaming.api.windowing.time.Time import org.apache.flink.streaming.api.scala.function.WindowFunction import org.apache.flink.streaming.api.windowing.windows.TimeWindow import org.apache.flink.api.common.functions.MapFunction import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.java.typeutils.RowTypeInfo import org.apache.flink.types.Row import org.apache.flink.api.common.serialization.SimpleStringSchema import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer011 import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer011 import org.apache.flink.streaming.connectors.kafka.KafkaSerializationSchema import org.apache.flink.streaming.connectors.kafka.KafkaContextAware import org.apache.flink.streaming.connectors.kafka.KafkaContextAware.KafkaContext import org.apache.flink.streaming.util.serialization.KeyedSerializationSchemaWrapper import java.util.Properties object MySQLCDC { def main(args: Array[String]): Unit = { val env = StreamExecutionEnvironment.getExecutionEnvironment env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime) val properties = new Properties() properties.setProperty("bootstrap.servers", "localhost:9092") properties.setProperty("group.id", "flink-group") val consumer = new FlinkKafkaConsumer011[String]("mysql-cdc", new SimpleStringSchema(), properties) val stream = env.addSource(consumer).map(new MapFunction[String, Row]() { override def map(value: String): Row = { val fields = value.split(",") Row.of(fields(0).toInt.asInstanceOf[Object], fields(1).asInstanceOf[Object], fields(2).asInstanceOf[Object]) } }).assignTimestampsAndWatermarks(new AssignerWithPunctuatedWatermarks[Row]() { override def extractTimestamp(row: Row, previousTimestamp: Long): Long = { row.getField(0).asInstanceOf[Int].toLong } override def checkAndGetNextWatermark(row: Row, extractedTimestamp: Long): Watermark = { new Watermark(extractedTimestamp) } }) val windowedStream = stream.keyBy(1).timeWindow(Time.seconds(10)).apply(new WindowFunction[Row, Row, Tuple, TimeWindow]() { override def apply(key: Tuple, window: TimeWindow, input: Iterable[Row], out: Collector[Row]): Unit = { val sortedInput = input.toList.sortBy(_.getField(0).asInstanceOf[Int]) val firstRow = sortedInput.head val lastRow = sortedInput.last out.collect(Row.of(firstRow.getField(1), firstRow.getField(2), lastRow.getField(2))) } }) val producer = new FlinkKafkaProducer011[String]("mysql-cdc-output", new KafkaSerializationSchema[String]() with KafkaContextAware[String] { var context: KafkaContext = _ override def serialize(element: String, timestamp: java.lang.Long): org.apache.kafka.clients.producer.ProducerRecord[Array[Byte], Array[Byte]] = { new org.apache.kafka.clients.producer.ProducerRecord(context.getOutputTopic(), element.getBytes()) } override def setRuntimeContext(context: KafkaContext): Unit = { this.context = context } }, properties, FlinkKafkaProducer011.Semantic.EXACTLY_ONCE) windowedStream.map(new MapFunction[Row, String]() { override def map(row: Row): String = { s"${row.getField(0)},${row.getField(1)},${row.getField(2)}" } }).addSink(producer) env.execute("MySQL CDC") } } ```

“相关推荐”对你有帮助么?

  • 非常没帮助
  • 没帮助
  • 一般
  • 有帮助
  • 非常有帮助
提交
评论 1
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值