第一步
依赖
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>5.1.38</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.11</artifactId>
<version>2.4.3</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>2.9.2</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-auth</artifactId>
<version>2.9.2</version>
</dependency>
drver编写
package com.baizhi.demo02
import com.mysql.jdbc.Driver
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.io.LongWritable
import org.apache.hadoop.mapreduce.lib.db.DBConfiguration
import org.apache.hadoop.mapreduce.lib.db.DBInputFormat
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.rdd.RDD
object SparkRDDMySQL {
def main(args: Array[String]): Unit = {
val sparkConf = new SparkConf()
.setMaster("local[6]")
.setAppName("wordcount")
val sc=new SparkContext(sparkConf)
val hadoopConfig = new Configuration()
DBConfiguration.configureDB(hadoopConfig,
"com.mysql.jdbc.Driver",
"jdbc:mysql://localhost:3306/test",
"root", "root")
hadoopConfig.set(DBConfiguration.INPUT_QUERY,"select id,name from t_user")
hadoopConfig.set(DBConfiguration.INPUT_COUNT_QUERY,"select count(*) from t_user")
hadoopConfig.set(DBConfiguration.INPUT_CLASS_PROPERTY,"com.baizhi.demo02.UserDBWriteable")
val userRDD:RDD[(LongWritable,UserDBWriteable)] = sc.newAPIHadoopRDD(hadoopConfig,classOf[DBInputFormat[UserDBWriteable]],classOf[LongWritable],classOf[UserDBWriteable])
userRDD.map(t=>(t._2.id,t._2.name))
.foreach(tuple=>println(tuple))
sc.stop();
}
}
package com.baizhi.demo02
import java.sql.{PreparedStatement, ResultSet}
import org.apache.hadoop.mapreduce.lib.db.DBWritable
class UserDBWriteable extends DBWritable{
var id:Int=_
var name:String=_
override def write(preparedStatement: PreparedStatement): Unit = {
}
override def readFields(rs: ResultSet): Unit = {
id = rs.getInt("id")
name=rs.getString("name")
}
}