Flink自定义Sink

示例:Flink写入MySQL

创建Maven项目

• 修改pom.xm文件

	<!- 添加以下依赖，根据自己使用的版本进行修改 ->
	<dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-scala_2.11</artifactId>
      <version>1.7.2</version>
    </dependency>
    <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-streaming-scala_2.11</artifactId>
      <version>1.7.2</version>
    </dependency>
    <dependency>
      <groupId>mysql</groupId>
      <artifactId>mysql-connector-java</artifactId>
      <version>5.1.46</version>
    </dependency>

实现源码

import java.sql.{Connection, DriverManager, PreparedStatement}
import org.apache.flink.configuration.Configuration
import org.apache.flink.streaming.api.functions.sink.RichSinkFunction
import org.apache.flink.streaming.api.scala._

//样例类student
case class student(id:Int,name:String)
object SinkMySQL {
  def main(args: Array[String]): Unit = {
  
  	//创建流数据执行环境
    val env: StreamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment

	//从本地读取文件数据
    val dataStream: DataStream[String] = env.readTextFile("in/students.txt")

	//将读取的数据进行转换成student类型
    val students: DataStream[student] = dataStream.map(x => {
      val strings: Array[String] = x.split(",")
      student(strings(0).toInt, strings(1))
    })

	//将转换后的数据sink到MySQL，MyRichSinkToMySQL是自己实现的类
    students.addSink(new MyRichSinkToMySQL)
	
	println("写入成功！")
	
    env.execute()
  }

}

//实现MyRichSinkToMySQL，继承RichSinkFunction(RichSinkFunction是富函数，可以设定生命周期，在open与close中实现连接与关闭MySQL连接)
class MyRichSinkToMySQL extends RichSinkFunction[student] {
  //预编译
  //mysql连接配置
  var conn: Connection =_
  //插入语句
  var insertTmp: PreparedStatement =_
  //更新语句
  var updateTmp: PreparedStatement =_

 //重写open方法
  override def open(parameters: Configuration): Unit = {
    //实现连接配置
    conn=DriverManager.getConnection("jdbc:mysql://192.168.**.**:3306/test","root","root123")
    //实现插入语句，在invoke中确实具体插入的数据内容
    insertTmp=conn.prepareStatement("insert into student (id,name) values (?,?)")
    //实现更新语句，在invoke中确实具体插入的数据内容
    updateTmp=conn.prepareStatement("update student set name = ? where id = ? ")
  }

   //实现invoke方法，执行具体的逻辑操作
  override def invoke(value: student): Unit = {
  	//执行更新操作
    updateTmp.setString(1,value.name)
    updateTmp.setInt(2,value.id)
    updateTmp.execute()

    //判断更新条数，如果为0，说明该学生信息不存在，即执行插入操作
    if (updateTmp.getUpdateCount == 0 ){
      insertTmp.setInt(1,value.id)
      insertTmp.setString(2,value.name)
      insertTmp.execute()
    }
  }
  
  //关闭所有连接
  override def close(): Unit = {   
    insertTmp.close()
    updateTmp.close()
    conn.close()
  }
}

student.txt文件数据

1,lj
2,ls
3,ww
1,zs

执行代码

查看MySQL中的student表