spark日志检查–将数据写入到数据中02

spark日志检查–将数据写入到数据中02

首先来看看链接MySQL的操作

引入的包是
import java.sql.{DriverManager, PreparedStatement, Connection}
    var conn: Connection = null
    var ps: PreparedStatement = null
    val sql = "INSERT INTO `person`( `num`, `ipdata`) VALUES (?,?)"
    try {
      conn = DriverManager.getConnection(connect_string)
      iterator.foreach(data => {
        ps = conn.prepareStatement(sql)
        ps.setString(1, data._1.toString)
        ps.setInt(2, data._2)
        ps.executeUpdate()
      }
      )
    } catch {
      case e: Exception => println("Mysql Exception")
    } finally {
      if (ps != null) {
        ps.close()
      }
      if (conn != null) {
        conn.close()
      }
    }

接下来是将实时的数据写到MySQL

object sparkHDFLdata {
  val user="root"
  val password = ""
  val host="localhost"
  val database="mytest"
  val connect_string = "jdbc:mysql://"+host +":3306/"+database+"?user="+user+"&password=" + password
  def main(args: Array[String]): Unit = {
  val sparkconf = new SparkConf().setAppName("hello").setMaster("local[4]")
  val scc = new StreamingContext(sparkconf,Seconds(10))
  val data =scc.textFileStream("file:Users/sicong/sparkStream/data/")
  val lines = data.flatMap(_.split("\n"))
  val dataOfMap = lines.map(x=>x.split(" ")).map(x=>(x(0),1))
  //将数据排序 -> 过滤  -> 移位
 val dataOfOut =dataOfMap.reduceByKey(_+_).map(x=>(x._2,x._1)).transform(
   x=>x.sortByKey(false)
 ).filter(x=>(x._1 > 50)).map( x=>(x._2,x._1))
//将每个分区数据写到 mysql
  dataOfOut.foreachRDD(
      x=>x.foreachPartition(y =>myFun(y))
    )
}