object SocketToMysql {
def main(args: Array[String]) {
val spark = SparkSession.builder().master("local[4]").appName("ToMysql")
.getOrCreate()
val kafka = spark.readStream
.format("socket")
.option("host", "x.x.x.x")
.option("port", 9999)
.load()
import spark.implicits._
val msg = kafka.selectExpr("CAST(value as String)")
val words = msg.as[String].map(_.split(",")).map(x => Weblog(x(0), x(1), x(2), x(3), x(4), x(5)))
val datetime = words.groupBy("datetime").count().toDF("value", "count")
val url = "jdbc:mysql://bigdata-pro04.mysql.com:3306/test"
val use = "root"
val password = "123456"
val write1 = new JDBCSink(url, use, password, "test", "timecount", "time", "count")
//
val update1 = datetime.writeStream.foreach(write1).outputMode("update").option("truncate", "false").start() //后面的阻塞了没有执行
val write = new JDBCSink(url, use, password, "test", "searchNameCount", "searchName", "count")
val searchname = words.groupBy("searchname").count().toDF("value", "count")
val update = searchname.writeStream.foreach(write).outputMode("update")
.option("truncate", "false")
.trigger(ProcessingTime("10 seconds"))
.start()
update.awaitTermination()
update1.awaitTermination()
}
}
控制台打印的日志: