自定义MySQL Source

<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>5.1.49</version>
</dependency>

package com.shujia.source

import org.apache.flink.streaming.api.functions.source.{ParallelSourceFunction, RichParallelSourceFunction, RichSourceFunction, SourceFunction}
import org.apache.flink.streaming.api.scala._

import java.sql.{Connection, DriverManager, ResultSet, Statement}

object Demo05MySQLSource {
  def main(args: Array[String]): Unit = {
    val env: StreamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment
    env
      .addSource(new MySQLSource)
      .print()

    env.execute()
  }

}

// 以MySQL作为数据源 构建MySQLSource
class MySQLSource extends SourceFunction[String] {
  override def run(ctx: SourceFunction.SourceContext[String]): Unit = {
    //加载驱动
    Class.forName("com.mysql.jdbc.Driver")

    //建立连接
    val conn: Connection = DriverManager.getConnection("jdbc:mysql://master:3306/student?useSSL=false", "root", "123456")

    // 创建Statement
    val stat: Statement = conn.createStatement()

    // 执行SQL
    val rs: ResultSet = stat.executeQuery("select id,name,age,gender,clazz from student")

    // 循环遍历返回的结果
    while (rs.next()) {
      val id: Int = rs.getInt("id")
      val name: String = rs.getString("name")
      val age: Int = rs.getInt("age")
      val gender: String = rs.getString("gender")
      val clazz: String = rs.getString("clazz")
      // 将获取到的每一条数据都发送到下游
      ctx.collect(s"$id,$name,$age,$gender,$clazz")
    }

    // 关闭连接
    conn.close()

  }

  override def cancel(): Unit = {

  }
}

package com.shujia.source

import org.apache.flink.configuration.Configuration
import org.apache.flink.streaming.api.functions.source.{ParallelSourceFunction, RichParallelSourceFunction, RichSourceFunction, SourceFunction}
import org.apache.flink.streaming.api.scala._

import java.sql.{Connection, DriverManager, ResultSet, Statement}

object Demo05MySQLSource {
  def main(args: Array[String]): Unit = {
    val env: StreamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment
    env.addSource(new MySQLSource).print()

    env.execute()
  }

}

/**
 * SourceFunction ：单线程
 * RichSourceFunction：多了open以及cancel两个方法
 * ParallelSourceFunction：并行的source
 * RichParallelSourceFunction：多了open以及cancel两个方法
 */

// 以MySQL作为数据源 构建MySQLSource
class MySQLSource extends RichParallelSourceFunction[String] {
  var conn:Connection = _
  /**
   * 在Flink程序运行时会执行一次，可以进行一些初始化操作
   * 会在run方法之前执行
   */
  override def open(parameters: Configuration): Unit = {
    println("建立连接")
    // 加载驱动
    Class.forName("com.mysql.jdbc.Driver")
    // 建立连接
    conn= DriverManager.getConnection("jdbc:mysql://master:3306/student?useSSL=false", "root", "123456")

  }

  /**
   * 一般在例如map、join等操作执行完成后会调用
   */
  override def close(): Unit = {


  }

  /**
   * 主要用于获取数据并发送到下游
   */
  override def run(ctx: SourceFunction.SourceContext[String]): Unit = {

    // 创建Statement
    val stat: Statement = conn.createStatement()

    // 执行SQL
    val rs: ResultSet = stat.executeQuery("select id,name,age,gender,clazz from student")

    // 循环遍历返回的结果
    while (rs.next()) {
      val id: Int = rs.getInt("id")
      val name: String = rs.getString("name")
      val age: Int = rs.getInt("age")
      val gender: String = rs.getString("gender")
      val clazz: String = rs.getString("clazz")
      // 将获取到的每一条数据都发送到下游
      ctx.collect(s"$id,$name,$age,$gender,$clazz")
    }

    // 关闭连接
    conn.close()

  }

  override def cancel(): Unit = {
    // 关闭连接
    conn.close()

  }
}