数据
sensor_1,1547718199,35.8
sensor_6,1547718201,15.4
sensor_7,1547718202,6.7
sensor_10,1547718205,38.1
sensor_1,1547718129,29.8
sensor_1,1547718158,5.8
sensor_1,1547718140,40.8
sensor_1,1547718111,11.8
package com.sinktest
import java.sql.{Connection, Driver, DriverManager, PreparedStatement}
import org.apache.flink.api.common.serialization.{SimpleStringEncoder, SimpleStringSchema}
import org.apache.flink.configuration.Configuration
import org.apache.flink.core.fs.Path
import org.apache.flink.streaming.api.functions.sink.{RichSinkFunction, SinkFunction}
import org.apache.flink.streaming.api.functions.sink.filesystem.StreamingFileSink
import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
import org.apache.flink.streaming.api.scala._
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer011
import org.apache.flink.streaming.connectors.redis.RedisSink
import org.apache.flink.streaming.connectors.redis.common.config.FlinkJedisPoolConfig
import org.apache.flink.streaming.connectors.redis.common.mapper.{RedisCommand, RedisCommandDescription, RedisMapper}
//定义样例类,温度传感器
case class SensorReading3(id:String,timestamp:Long,temperature:Double)
object FileSink {
def main(args: Array[String]): Unit = {
val env=StreamExecutionEnvironment.getExecutionEnvironment
val inputPath ="D:\\workspace\\ideastudy\\flinkstudy\\src\\main\\scala\\com\\apitest\\sensor.txt"
env.setParallelism(1)//为了测试保持读取数据的顺序
val inputStream = env.readTextFile(inputPath)
//1.先转换成样例类类型
val dataStream= inputStream.map(data=>{
val arr = data.split(",")
SensorReading3(arr(0),arr(1).toLong,arr(2).toDouble)
})
//1.输出到文件
//dataStream.writeAsText("D:\\workspace\\ideastudy\\flinkstudy\\src\\main\\resources\\hello2.txt")
//2.文件写入第二种方式
// dataStream.addSink(StreamingFileSink.forRowFormat(
// new Path("D:\\workspace\\ideastudy\\flinkstudy\\src\\main\\resources\\hello3.txt"),
// new SimpleStringEncoder[SensorReading3]()
// ).build()
// )
//3.往kafka中写数据
val kafkaSink=dataStream.map(data=>data.toString)
//kafkaSink.addSink(new FlinkKafkaProducer011[String]("localhost:9092","sinktest",new SimpleStringSchema()))
//4.往redis中写数据
val conf = new FlinkJedisPoolConfig.Builder()
.setHost("localhost")
.setPort(6379)
.build()
// dataStream.addSink(new RedisSink[SensorReading3](conf,new MyRedisMapper))
//5.自定义sink
dataStream.addSink(new MyJdbcSinkFunc)
env.execute("file sink test")
}
}
//定义一个RedisMapper
class MyRedisMapper extends RedisMapper[SensorReading3]{
//定义保存数据写入redis的命令
override def getCommandDescription: RedisCommandDescription = {
//Hset 表示为hash表 ,第二个参数表示为表名
new RedisCommandDescription(RedisCommand.HSET,"sensor_temp")
}
//将温度值指定为value
override def getKeyFromData(t: SensorReading3): String = {
t.temperature.toString
}
//将id指定为key
override def getValueFromData(t: SensorReading3): String = {
t.id
}
}
class MyJdbcSinkFunc extends RichSinkFunction[SensorReading3]{
//定义连接、预编译语句
var conn:Connection= _
var insertStmt:PreparedStatement=_
var updateStmt:PreparedStatement=_
override def open(parameters: Configuration): Unit = {
conn=DriverManager.getConnection("jdbc:mysql://localhost/test","root","root")
insertStmt=conn.prepareStatement("insert into test(id,temp) values(?,?)")
updateStmt=conn.prepareStatement("update test set temp=? where id=?")
}
//每来一条数据走这个方法
override def invoke(value: SensorReading3, context: SinkFunction.Context[_]): Unit = {
//先执行更新操作,查到就更新
updateStmt.setDouble(1,value.temperature);
updateStmt.setString(2,value.id);
updateStmt.execute();
if(updateStmt.getUpdateCount == 0){
insertStmt.setString(1,value.id);
insertStmt.setDouble(2,value.temperature);
insertStmt.execute();
}
}
override def close(): Unit = {
insertStmt.close()
updateStmt.close()
conn.close()
}
}