查询mysql数据库结果,生成一个rdd
需要导入依赖
mysql-connector-java这个依赖
package com.qf.gp1921.day12
import java.sql.DriverManager
import org.apache.spark.rdd.JdbcRDD
import org.apache.spark.{SparkConf, SparkContext}
object JDBCRDDTest {
def main(args: Array[String]): Unit = {
val conf = new SparkConf().setAppName("JDBCRDDTest").setMaster("local[2]")
val sc = new SparkContext(conf)
val sql = "select id, location, counts, access_date from location_info where id >= ? and id <= ?"
// jdbc连接驱动设置
val jdbcurl = "jdbc:mysql://node03:3306/bigdata?useUnicode=true&charsetEncoding=utf8"
val user = "root"
val password = "root"
// 获取数据库的连接
val conn = () => {
Class.forName("com.mysql.jdbc.Driver").newInstance()
DriverManager.getConnection(jdbcurl, user, password)
}
// 调用JdbcRDD获取mysql的数据
val jdbcRDD = new JdbcRDD(
sc,
conn,
sql,
0, //sql中的第一个问号传入的参数
1000, //sql中的第二个问号传入的参数
1,
res => {
val id = res.getInt("id")
val location = res.getString("location")
val counts = res.getInt("counts")
val access_date = res.getDate("access_date")
//上面括号中的是mysql中的字段名
(id, location, counts, access_date)
}
)
jdbcRDD.foreach(println)
sc.stop()
}
}