spark日志检查–将数据写入到数据中02
首先来看看链接MySQL的操作
引入的包是
import java.sql.{DriverManager, PreparedStatement, Connection}
var conn: Connection = null
var ps: PreparedStatement = null
val sql = "INSERT INTO `person`( `num`, `ipdata`) VALUES (?,?)"
try {
conn = DriverManager.getConnection(connect_string)
iterator.foreach(data => {
ps = conn.prepareStatement(sql)
ps.setString(1, data._1.toString)
ps.setInt(2, data._2)
ps.executeUpdate()
}
)
} catch {
case e: Exception => println("Mysql Exception")
} finally {
if (ps != null) {
ps.close()
}
if (conn != null) {
conn.close()
}
}
接下来是将实时的数据写到MySQL
object sparkHDFLdata {
val user="root"
val password = ""
val host="localhost"
val database="mytest"
val connect_string = "jdbc:mysql://"+host +":3306/"+database+"?user="+user+"&password=" + password
def main(args: Array[String]): Unit = {
val sparkconf = new SparkConf().setAppName("hello").setMaster("local[4]")
val scc = new StreamingContext(sparkconf,Seconds(10))
val data =scc.textFileStream("file:Users/sicong/sparkStream/data/")
val lines = data.flatMap(_.split("\n"))
val dataOfMap = lines.map(x=>x.split(" ")).map(x=>(x(0),1))
//将数据排序 -> 过滤 -> 移位
val dataOfOut =dataOfMap.reduceByKey(_+_).map(x=>(x._2,x._1)).transform(
x=>x.sortByKey(false)
).filter(x=>(x._1 > 50)).map( x=>(x._2,x._1))
//将每个分区数据写到 mysql
dataOfOut.foreachRDD(
x=>x.foreachPartition(y =>myFun(y))
)
}