sparkSession:
/**
* The entry point to programming Spark with the Dataset and DataFrame API.
*
* In environments that this has been created upfront (e.g. REPL, notebooks), use the builder
* to get an existing session:
*
* {{{
* SparkSession.builder().getOrCreate()
* }}}
*
* The builder can also be used to create a new session:
*
* {{{
* SparkSession.builder
* .master("local")
* .appName("Word Count")
* .config("spark.some.config.option", "some-value")
* .getOrCreate()
* }}}
*
* @param sparkContext The Spark context associated with this Spark session.
* @param existingSharedState If supplied, use the existing shared state
* instead of creating a new one.
* @param parentSessionState If supplied, inherit all session state (i.e. temporary
* views, SQL config, UDFs etc) from parent.
*/
@InterfaceStability.Stable
class SparkSession private(
@transient val sparkContext: SparkContext,
@transient private val existingSharedState: Option[SharedState],
@transient private val parentSessionState: Option[SessionState],
@transient private[sql] val extensions: SparkSessionExtensions)
extends Serializable with Closeable with Logging { self =>
private[sql] def this(sc: SparkContext) {
this(sc, None, None, new SparkSessionExtensions)
}
sparkContext.assertNotStopped()
// If there is no active SparkSession, uses the default SQL conf. Otherwise, use the session's.
SQLConf.setSQLConfGetter(() => {
SparkSession.getActiveSession.map(_.sessionState.conf).getOrElse(SQLConf.getFallbackConf)
})
……
package com.ljt.spark01.sql
import org.apache.spark.SparkConf
import org.apache.spark.sql.SparkSession
import com.ljt.spark01.sql.TestActor.Bean
import com.ljt.spark01.sql.TestActor.CreateView
import akka.actor.Actor
import akka.actor.ActorLogging
import scala.util.Random
import org.apache.spark.SparkConf
import akka.actor.ActorSystem
import akka.actor.Props
import org.apache.spark.SparkConf
/**
* ljt
* 我的服务端的逻辑是在actor内部进行的,但发现多个actor中执行的过程中,访问到了其他actor内部session中注册的临时表
*
*/
class TestActor(sparkConf: SparkConf) extends Actor with ActorLogging {
var sparkSession: SparkSession = _
override def preStart(): Unit = {
sparkSession=getSparkSession(sparkConf)
}
def getSparkSession(sparkConf: SparkConf) = SparkSession.synchronized {
SparkSession.clearDefaultSession()
val session = SparkSession.builder().config(sparkConf).getOrCreate()
SparkSession.clearDefaultSession()
session
}
override def postStop(): Unit = sparkSession.stop()
override def receive: Receive = {
case CreateView(limit) ⇒ sparkSession.createDataFrame(getBeans(limit)).createOrReplaceTempView(“test1”)
case s: String ⇒
log.info(s”exec $s”)
sparkSession.sql(s).show(1000)
case e: Any ⇒ println(e)
}
val ids = 1 to 1000
val names = (‘a’ to ‘z’).map(_.toString())
val beans = for {
id <- ids
name <- names
} yield Bean(id, name)
def getBeans(limit: Int): Seq[Bean] = {
Random.shuffle(beans).take(limit)
}
}
//伴生对象
object TestActor {
case class CreateView(limit: Int)
case class Bean(id: Int, name: String)
}
/**
* 测试驱动类
*/
object test {
def main(args: Array[String]): Unit = {
val saprkConf = new SparkConf().setAppName(“testActor”).setMaster(“local[*]”)
val actorSystem = ActorSystem(“testackkaSystem”)
val props = Props(new TestActor(saprkConf))
val actor1 = actorSystem.actorOf(props, name = “actor01”)
val actor2 = actorSystem.actorOf(props, name = “actor02”)
val actor3 = actorSystem.actorOf(props, name = “actor03”)
val actor4 = actorSystem.actorOf(props, name = “actor04”)
val actor5 = actorSystem.actorOf(props, name = “actor05”)
/**
* 答疑:参考http://blog.csdn.net/wust__wangfan/article/details/50911023
* Null和Nil
* Null是所有AnyRef的子类,在scala的类型系统中,AnyRef是Any的子类,同时Any子类的还有AnyVal。对应java值类型的所有类型都是AnyVal的子类。所以Null可以赋值给所有的引用类型(AnyRef),不能赋值给值类型,这个java的语义是相同的。
* null是Null的唯一对象
* Nil是一个空的List,定义为List[Nothing],根据List的定义List[+A],所有Nil是所有List[T]的子类。
*/
val actors = actor1 :: actor2 :: actor3 :: actor4 :: actor5 :: Nil
def getLimit = Random.nextInt(100)
actors.foreach { x => x ! CreateView(getLimit) }
Thread sleep 1000
// actors.foreach(_ ! “select * from test1”)
actors.foreach { x => x ! “show tables” }
Thread sleep 1000
actors.foreach { x => x ! "select * from test1" }
Thread sleep 1000
}
}