Spark Core之RDD原理
RDD(Resilent Distrutbed DataSet): 弹性分布式数据集,是Spark中最为核心的抽象,代表一个不可变,可分区,支持并行计算的数据集合。
RDD的创建
通过Scala(Seq)集合
import org.apache.spark.{
SparkConf, SparkContext}
object CreateRDDWithCollection {
def main(args: Array[String]): Unit = {
val conf = new SparkConf().setAppName("rdd create").setMaster("local[3]")
val sc = new SparkContext(conf)
// 通过集合构建RDD
// 分区和任务并行度关系
/*
p0:
Hello Scala (Hello,1) (Scala,1) (Hello,1) (Spark,1)
Hello Spark
p1:
Hello Hello Hello
*/
// 方式1
// val rdd = sc.parallelize(List("Hello Scala","Hello Spark","Hello Hello Hello"),2)
// 方式2
val rdd = sc.makeRDD(List("Hello Scala","Hello Spark","Hello Hello Hello"),2)
rdd
.flatMap(_.split(" "))
.map((_,1))
.groupByKey() // shuffle
.map(t2 => (t2._1,t2._2.size))
.foreach(println)
sc.stop()
}
}
通过文件系统
- Local
package com.baizhi.datasource
import org.apache.spark.{
SparkConf, SparkContext}
object CreateRDDWithLocalFileSystem {
def main(args: Array[String]): Unit = {
val conf = new SparkConf().setAppName("rdd create").setMaster("local[3]")
val sc = new SparkContext(conf)
// 通过local fs构建RDD
// 分区和任务并行度关系
//--------------------------------------------------------
// 方式1
// 返回值RDD[String: 一行记录]
// val rdd = sc.textFile("file:///d://README.txt")
//
// rdd
// .flatMap(_.split(" "))
// .map((_,1))
// .groupByKey() // shuffle
// .map(t2 => (t2._1,t2._2.size))
// .foreach(println)
//--------------------------------------------------------
// 方式2
// 返回值RDD(path,完整内容)
val rdd = sc.wholeTextFiles("file:///d://README.txt")
// rdd.foreach(t2 => println(t2._1 +"\t" + t2._2))
rdd
.map(t2 => t2._2) // Tuple2 --> String
.flatMap(wholeText => wholeText.split("\n")) // 文本 --> line
.flatMap(line => line.split(" "))
.map((_,1))
.groupByKey() // shuffle
.map(t2 => (t2._1, t2._2.size))
.foreach(println)
sc.stop()
}
}
- HDFS
确保HDFS集群服务正常
import org.apache.spark.{
SparkConf, SparkContext}
object CreateRDDWithHDFS {
def main(args: Array[String]): Unit = {
val conf = new SparkConf().setAppName("rdd create").setMaster("local[3]")
val sc = new SparkContext(conf)
// 通过local fs构建RDD
// 分区和任务并行度关系
//--------------------------------------------------------
// 方式1
// 返回值RDD[String: 一行记录]
// val rdd = sc.textFile("hdfs://SparkOnStandalone:9000/README.md")
//
// rdd
// .flatMap(_.split(" "))
// .map((_, 1))
// .groupByKey() // shuffle
// .map(t2 => (t2._1, t2._2.size))
// .foreach(println)
//--------------------------------------------------------
// 方式2
// 返回值RDD(path,完整内容)
val rdd = sc.wholeTextFiles("hdfs://SparkOnStandalone:9000/README.md")
// rdd.foreach(t2 => println(t2._1 +"\t" + t2._2))
rdd
.map(t2 => t2._2) // Tuple2 --> String
.flatMap(wholeText => wholeText.split("\n")) // 文本 --> line
.flatMap(line => line.split(" "))
.map((_, 1))
.groupByKey() // shuffle
.map(t2 => (t2._1, t2._2.size))
.foreach(println)
sc.stop()
}
}
通过RDBMS
通过MySQL数据库表构建Spark核心抽象RDD
user表—> RDD
import java.sql.DriverManager
import com.mysql.jdbc.Driver
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.io.{
IntWritable, LongWritable}
import org.apache.hadoop.mapreduce.lib.db.{
DBConfiguration, DBInputFormat}
import org.apache.spark.rdd.JdbcRDD
import org.apache.spark.{
SparkConf, SparkContext}
/**
* 通过RDBMS创建RDD
*/
object CreateRDDWithRDBMS {
def main(args: Array[String]): Unit = {
val conf = new SparkConf().setMaster("local[*]").setAppName("rdd with rdbms")
val sc = new SparkContext(conf)
// 方法一
// m1(sc)
// 方法二
m2(sc)
sc.stop()
}
def m1(sc: SparkContext) = {
// 创建hadoop配置对象 DBInputFormat
val hadoopConf = new Configuration()
// 数据库的连接参数
hadoopConf.set(DBConfiguration.DRIVER_CLASS_PROPERTY, "com.mysql.jdbc.Driver")
hadoopConf.set(DBConfiguration.URL_PROPERTY, "jdbc:mysql://localhost:3306/hadoop")
hadoopConf.set(DBConfiguration.PASSWORD_PROPERTY, "1234")
hadoopConf.set(DBConfiguration.USERNAME_PROPERTY, "root")
hadoopConf.set(DBConfiguration.INPUT_QUERY, "select * from user ")
hadoopConf.set(DBConfiguration.INPUT_COUNT_QUERY, "select count(*) from user")
// 将自定义序列化对象定义配置对象中
hadoopCo