flink 将mysql作为Source和Sink的代码示例

本文介绍了如何在Flink中使用Java实现从MySQL读取数据(Source)并写入到MySQL(Sink)的操作。首先,通过maven引入相关依赖,然后详细展示了SourceFromMySQL和SinkToMySQL的工具类代码,最后在主函数中演示了如何使用这些工具类。
摘要由CSDN通过智能技术生成

1.maven导入

<dependency>
			<groupId>mysql</groupId>
			<artifactId>mysql-connector-java</artifactId>
			<version>5.1.34</version>
		</dependency>

2.SourceFromMySQL工具类java代码

import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.functions.source.RichSourceFunction;

import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;

/**
 * @Description TODO
 * @Author zytshijack
 * @Date 2019-05-14 08:32
 * @Version 1.0
 */
public class SourceFromMySQL extends RichSourceFunction<Student> { //Student就是一个自定义类

    PreparedStatement ps;
    private Connection connection;

    /**
     * open() 方法中建立连接,这样不用每次 invoke 的时候都要建立连接和释放连接。
     *
     * @param parameters
     * @throws Exception
     */
    @Override
    public void open(Configuration parameters) throws E
以下是使用 Flink 实现 MySQL CDC 的 Scala 代码示例: ```scala import org.apache.flink.streaming.api.scala._ import org.apache.flink.streaming.api.functions.source.SourceFunction import org.apache.flink.streaming.api.functions.AssignerWithPunctuatedWatermarks import org.apache.flink.streaming.api.watermark.Watermark import org.apache.flink.streaming.api.functions.sink.SinkFunction import org.apache.flink.streaming.api.functions.ProcessFunction import org.apache.flink.util.Collector import org.apache.flink.streaming.api.TimeCharacteristic import org.apache.flink.streaming.api.windowing.time.Time import org.apache.flink.streaming.api.scala.function.WindowFunction import org.apache.flink.streaming.api.windowing.windows.TimeWindow import org.apache.flink.api.common.functions.MapFunction import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.java.typeutils.RowTypeInfo import org.apache.flink.types.Row import org.apache.flink.api.common.serialization.SimpleStringSchema import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer011 import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer011 import org.apache.flink.streaming.connectors.kafka.KafkaSerializationSchema import org.apache.flink.streaming.connectors.kafka.KafkaContextAware import org.apache.flink.streaming.connectors.kafka.KafkaContextAware.KafkaContext import org.apache.flink.streaming.util.serialization.KeyedSerializationSchemaWrapper import java.util.Properties object MySQLCDC { def main(args: Array[String]): Unit = { val env = StreamExecutionEnvironment.getExecutionEnvironment env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime) val properties = new Properties() properties.setProperty("bootstrap.servers", "localhost:9092") properties.setProperty("group.id", "flink-group") val consumer = new FlinkKafkaConsumer011[String]("mysql-cdc", new SimpleStringSchema(), properties) val stream = env.addSource(consumer).map(new MapFunction[String, Row]() { override def map(value: String): Row = { val fields = value.split(",") Row.of(fields(0).toInt.asInstanceOf[Object], fields(1).asInstanceOf[Object], fields(2).asInstanceOf[Object]) } }).assignTimestampsAndWatermarks(new AssignerWithPunctuatedWatermarks[Row]() { override def extractTimestamp(row: Row, previousTimestamp: Long): Long = { row.getField(0).asInstanceOf[Int].toLong } override def checkAndGetNextWatermark(row: Row, extractedTimestamp: Long): Watermark = { new Watermark(extractedTimestamp) } }) val windowedStream = stream.keyBy(1).timeWindow(Time.seconds(10)).apply(new WindowFunction[Row, Row, Tuple, TimeWindow]() { override def apply(key: Tuple, window: TimeWindow, input: Iterable[Row], out: Collector[Row]): Unit = { val sortedInput = input.toList.sortBy(_.getField(0).asInstanceOf[Int]) val firstRow = sortedInput.head val lastRow = sortedInput.last out.collect(Row.of(firstRow.getField(1), firstRow.getField(2), lastRow.getField(2))) } }) val producer = new FlinkKafkaProducer011[String]("mysql-cdc-output", new KafkaSerializationSchema[String]() with KafkaContextAware[String] { var context: KafkaContext = _ override def serialize(element: String, timestamp: java.lang.Long): org.apache.kafka.clients.producer.ProducerRecord[Array[Byte], Array[Byte]] = { new org.apache.kafka.clients.producer.ProducerRecord(context.getOutputTopic(), element.getBytes()) } override def setRuntimeContext(context: KafkaContext): Unit = { this.context = context } }, properties, FlinkKafkaProducer011.Semantic.EXACTLY_ONCE) windowedStream.map(new MapFunction[Row, String]() { override def map(row: Row): String = { s"${row.getField(0)},${row.getField(1)},${row.getField(2)}" } }).addSink(producer) env.execute("MySQL CDC") } } ```
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值