SecurityManager 安全管理器
在初始化SparkContext是会创建SparkEnv,而创建SparkEnv时会创建SecurityManager
在默认情况下spark安全性处于关闭状态,这当然时不可取的,所以创建SecurityManager来保证spark的安全
源码清单和我的理解注释
创建初始变量
// allow all users/groups to have view/modify permissions
// 通配符,允许所有用户或群组有去查看或修改的权限
private val WILDCARD_ACL = "*"
// 在sparkConf中查找spark.authenticate的值,若sparkConf没有则为false
// 通过设置spark.authenticate参数决定是否使用Spark共享密钥的进行身份认证,默认为false
private val authOn = sparkConf.getBoolean(SecurityManager.SPARK_AUTH_CONF, false)
// keep spark.ui.acls.enable for backwards compatibility with 1.0
// 与之前版本前兼容的属性
private var aclsOn =
sparkConf.getBoolean("spark.acls.enable", sparkConf.getBoolean("spark.ui.acls.enable", false))
// admin acls should be set before view or modify acls
// acls(access control lists)访问控制列表
// 在查看或修改acl前应该设置管理员acl
// adminAcls为管理员账号列表
private var adminAcls: Set[String] =
stringToSet(sparkConf.get("spark.admin.acls", ""))
// admin group acls should be set before view or modify group acls
// 在查看或修改管理员组的acl前应该设置管理员的acl
// adminAclsGroups为管理员账号组的列表
private var adminAclsGroups : Set[String] =
stringToSet(sparkConf.get("spark.admin.acls.groups", ""))
// 拥有查看acl权限的账号列表
private var viewAcls: Set[String] = _
// 拥有查看acl账号组权限的列表
private var viewAclsGroups: Set[String] = _
// list of users who have permission to modify the application. This should
// apply to both UI and CLI for things like killing the application.
// 有修改权限的用户账号列表
private var modifyAcls: Set[String] = _
//有修改权限的用户组账号列表
private var modifyAclsGroups: Set[String] = _
// always add the current user and SPARK_USER to the viewAcls
// 默认用户(user.name或者是当前系统的用户)
private val defaultAclUsers = Set[String](System.getProperty("user.name", ""),
Utils.getCurrentUserName())
// 将默认用户加入setViewAcls、setModifyAcls、setViewAclsGroups和setModifyAclsGroups中
setViewAcls(defaultAclUsers, sparkConf.get("spark.ui.view.acls", ""))
setModifyAcls(defaultAclUsers, sparkConf.get("spark.modify.acls", ""))
setViewAclsGroups(sparkConf.get("spark.ui.view.acls.groups", ""));
setModifyAclsGroups(sparkConf.get("spark.modify.acls.groups", ""));
根据不同模式生成密钥
1.对于YARN上的 Spark 和本地部署,Spark将自动处理共享密钥的生成和分发。每个应用程序将使用唯一的共享密钥
2.其他模式,spark.authenticate.secret必须在每个节点上进行配置。密钥将由所有节点和应用程序共享,但不安全。
private val secretKey = generateSecretKey()
def isAuthenticationEnabled(): Boolean = authOn
private def generateSecretKey(): String = {
// 若不适用Spark共享密钥的进行身份认证,则返回null
if (!isAuthenticationEnabled) {
null
}
//若是spark on yarn模式 ,生成密钥
else if (SparkHadoopUtil.get.isYarnMode) {
// In YARN mode, the secure cookie will be created by the driver and stashed in the
// user's credentials, where executors can get it. The check for an array of size 0
// is because of the test code in YarnSparkHadoopUtilSuite.
//在yarn模式下,dirver创建cookie并保存在用户凭证中,executors 可以从中获取
val secretKey = SparkHadoopUtil.get.getSecretKeyFromUserCredentials(SECRET_LOOKUP_KEY)
if (secretKey == null || secretKey.length == 0) {
logDebug("generateSecretKey: yarn mode, secret key from credentials is null")
val rnd = new SecureRandom()
val length = sparkConf.getInt("spark.authenticate.secretBitLength", 256) / JByte.SIZE
val secret = new Array[Byte](length)
rnd.nextBytes(secret)
val cookie = HashCodes.fromBytes(secret).toString()
SparkHadoopUtil.get.addSecretKeyToUserCredentials(SECRET_LOOKUP_KEY, cookie)
cookie
} else {
new Text(secretKey).toString
}
}
// 其他模式下,须在配置SecurityManager.ENV_AUTH_SECRET或者是SecurityManager.SPARK_AUTH_SECRET_CONF中获取
else {
// user must have set spark.authenticate.secret config
// For Master/Worker, auth secret is in conf; for Executors, it is in env variable
Option(sparkConf.getenv(SecurityManager.ENV_AUTH_SECRET))
.orElse(sparkConf.getOption(SecurityManager.SPARK_AUTH_SECRET_CONF)) match {
case Some(value) => value
case None =>
throw new IllegalArgumentException(
"Error: a secret key must be specified via the " +
SecurityManager.SPARK_AUTH_SECRET_CONF + " config")
}
}
}
通过匿名内部类的方法实现Authenticator,生成验证口令,
// Set our own authenticator to properly negotiate user/password for HTTP connections.
// This is needed by the HTTP client fetching from the HttpServer. Put here so its
// only set once.
//设置我们自己的身份验证器以正确协商http连接的用户/密码。
//这是http客户端从httpserver获取数据所需要的。把它放在这里
//只设置一次。
if (authOn) {
Authenticator.setDefault(
// 创建口令认证实例,复写PasswordAuthentication方法,获得用户名和密码
new Authenticator() {
override def getPasswordAuthentication(): PasswordAuthentication = {
var passAuth: PasswordAuthentication = null
val userInfo = getRequestingURL().getUserInfo()
if (userInfo != null) {
val parts = userInfo.split(":", 2)
passAuth = new PasswordAuthentication(parts(0), parts(1).toCharArray())
}
return passAuth
}
}
)
}