Spark+Scalar+Mysql

包：mysql-connector-java-5.1.39-bin.jar

平台：Win8.1

环境：MyEclipse2015

hadoop-2.7.3.tar.gz + winutils.exe

spark-2.1.0-bin-hadoop2.7.tgz

scala-2.11.8.tgz

package org.ah.rcmd
import java.sql.{ DriverManager, PreparedStatement, Connection }

object Rdbms {
  def setRcmdResult(iterator: Iterator[(String, String, String, String, String, String)]): Unit = {
    var conn: Connection = null
    var ps: PreparedStatement = null
    val sbSqlUpdate = new StringBuilder;
    sbSqlUpdate ++= " UPDATE recommond"
    sbSqlUpdate ++= " SET"
    sbSqlUpdate ++= "     url1 = ?,"
    sbSqlUpdate ++= "     url2 = ?,"
    sbSqlUpdate ++= "     url3 = ?,"
    sbSqlUpdate ++= "     url4 = ?,"
    sbSqlUpdate ++= "     url5 = ?"
    sbSqlUpdate ++= " WHERE uName = ?"

    val sqlInsert = "insert into recommond(url1,url2,url3,url4,url5,uName) values (?,?,?,?,?,?)"
    try {
      conn = DriverManager.getConnection("jdbc:mysql://localhost:3306/ahrecommend",
        "root", "root")
      iterator.foreach(data => {
        ps = conn.prepareStatement(sbSqlUpdate.toString)
        ps.setString(1, data._2)
        ps.setString(2, data._3)
        ps.setString(3, data._4)
        ps.setString(4, data._5)
        ps.setString(5, data._6)
        ps.setString(6, data._1)
        if (ps.executeUpdate < 1) {
          ps = conn.prepareStatement(sqlInsert)
          ps.setString(1, data._2)
          ps.setString(2, data._3)
          ps.setString(3, data._4)
          ps.setString(4, data._5)
          ps.setString(5, data._6)
          ps.setString(6, data._1)
          ps.executeUpdate()
        }
      })
    } catch {
      case e: Exception => println(e.fillInStackTrace())
    } finally {
      if (ps != null) {
        ps.close()
        ps = null
      }
      if (conn != null) {
        conn.close()
        conn = null
      }
    }
  }
}

package org.ah.rcmd
import org.apache.spark.{ SparkContext, SparkConf }

object RDDtoMysql {
  def main(args: Array[String]) {
    val conf = new SparkConf().setAppName("RDDToMysql").setMaster("local")
    val sc = new SparkContext(conf)
    // 获取数据
    val data = sc.parallelize(List(("Andy", "W1", "W2", "W3", "W4", "W5"), ("Cat", "W1", "W1", "W1", "W1", "W1")))
    // 写入数据库
    data.foreachPartition(Rdbms.setRcmdResult)
  }
}

 

转载于:https://www.cnblogs.com/AndyHoo/p/6687242.html