先建表
数据
sensor_1, 1547718199, 35.80018327300259
sensor_6, 1547718201, 15.402984393403084
sensor_7, 1547718202, 6.720945201171228
sensor_10, 1547718205, 38.101067604893444
sensor_1, 1547718206, 35.1
sensor_1, 1547718207, 35.6
结果
package com.flink.sourceAndSink.sink
import java.sql.{Connection, DriverManager, PreparedStatement}
import com.flink.sourceAndSink.SensorReading
import org.apache.flink.configuration.Configuration
import org.apache.flink.streaming.api.functions.sink.{RichSinkFunction, SinkFunction}
import org.apache.flink.streaming.api.scala.{DataStream, StreamExecutionEnvironment}
/**
* Created by Shi shuai RollerQing on 2019/12/19 15:16
*/
object JdbcSink {
def main(args: Array[String]): Unit = {
val env = StreamExecutionEnvironment.getExecutionEnvironment
env.setParallelism(1)
import org.apache.flink.api.scala._
//1.读取文件到kafka的sinkTest的topic 一般只是测试使用
val streamFromFile: DataStream[String] = env.readTextFile("C:\\Users\\HP\\IdeaProjects\\sparkCore\\flink\\src\\main\\resources\\sensor.txt")
val dataStream: DataStream[SensorReading] = streamFromFile.map(data => {
val dataArray = data.split(",")
SensorReading(dataArray(0).trim, dataArray(1).trim.toLong, dataArray(2).trim.toDouble)
})
//sink
dataStream.addSink(new MyJdbcSink())
env.execute("mysql sink test")
}
}
//因为SinkFunction的功能比较少 所以使用rich的SinkFunction
class MyJdbcSink() extends RichSinkFunction[SensorReading] {
//定义sql连接、预编译器
var conn: Connection = _
var insertStmt: PreparedStatement = _
var updateStmt: PreparedStatement = _
//初始化 、 创建连接 、 和 预编译语句
override def open(parameters: Configuration): Unit = {
super.open(parameters)
conn = DriverManager.getConnection("jdbc:mysql://hadoop01:3306/test", "root", "root")
insertStmt = conn.prepareStatement("insert into temperatures (sensor, temp) values (? , ?)")
updateStmt = conn.prepareStatement("update temperatures set temp = ? where sensor = ?")
}
// 调用连接 执行sql
override def invoke(value: SensorReading, context: SinkFunction.Context[_]): Unit = {
// 执行更新语句
updateStmt.setDouble(1, value.temperature)
updateStmt.setString(2, value.id)
updateStmt.execute()
//如果update没有更新 即 没有查询到数据 即 没有该id 那么执行插入
if (updateStmt.getUpdateCount == 0) {
insertStmt.setString(1, value.id)
insertStmt.setDouble(2, value.temperature)
insertStmt.execute()
}
}
//关闭时做清理工作
override def close(): Unit = {
insertStmt.close()
updateStmt.close()
conn.close()
}
}