Hadoop的客户端是通过FileSystem类操作hdfs的。
FileSystem.get()方法获取FileSystem对象。
public static FileSystem get(final URI uri, final Configuration conf,
final String user) throws IOException,InterruptedException {
String ticketCachePath =
conf.get(CommonConfigurationKeys.KERBEROS_TICKET_CACHE_PATH);
UserGroupInformationugi =
UserGroupInformation.getBestUGI(ticketCachePath, user);
return ugi.doAs(newPrivilegedExceptionAction<FileSystem>() {
@Override
public FileSystem run() throwsIOException {
return get(uri, conf);
}
});
}
在标红色的这一行中会获取用户名。
public static UserGroupInformation getBestUGI(
String ticketCachePath, String user)throws IOException {
if (ticketCachePath != null) {
returngetUGIFromTicketCache(ticketCachePath, user);
} else if (user == null) {
return getCurrentUser();
} else {
return createRemoteUser(user);
}
}
@InterfaceAudience.Public
@InterfaceStability.Evolving
public synchronized
static UserGroupInformation getCurrentUser() throws IOException {
AccessControlContext context =AccessController.getContext();
Subject subject =Subject.getSubject(context);
if (subject == null ||subject.getPrincipals(User.class).isEmpty()) {
return getLoginUser();
} else {
return new UserGroupInformation(subject);
}
}
@InterfaceAudience.Public
@InterfaceStability.Evolving
public synchronized
static UserGroupInformation getLoginUser() throws IOException {
if (loginUser == null) {
loginUserFromSubject(null);
}
return loginUser;
}
@InterfaceAudience.Public
@InterfaceStability.Evolving
public synchronized
static void loginUserFromSubject(Subject subject) throws IOException {
ensureInitialized();
try {
if (subject == null) {
subject = new Subject();
}
LoginContextlogin =
newLoginContext(authenticationMethod.getLoginAppName(),
subject, newHadoopConfiguration());//关键代码1
login.login();//关键代码2
UserGroupInformation realUser = newUserGroupInformation(subject);
realUser.setLogin(login);
realUser.setAuthenticationMethod(authenticationMethod);
realUser = newUserGroupInformation(login.getSubject());
// If the HADOOP_PROXY_USER environmentvariable or property
// is specified, create a proxy user asthe logged in user.
String proxyUser = System.getenv(HADOOP_PROXY_USER);
if (proxyUser == null) {
proxyUser =System.getProperty(HADOOP_PROXY_USER);
}
loginUser = proxyUser == null ? realUser: createProxyUser(proxyUser, realUser);
String fileLocation =System.getenv(HADOOP_TOKEN_FILE_LOCATION);
if (fileLocation != null) {
// Load the token storage file and putall of the tokens into the
// user. Don't use the FileSystem APIfor reading since it has a lock
// cycle (HADOOP-9212).
Credentials cred =Credentials.readTokenStorageFile(
new File(fileLocation), conf);
loginUser.addCredentials(cred);
}
loginUser.spawnAutoRenewalThreadForUserCreds();
} catch (LoginException le) {
LOG.debug("failure to login",le);
throw new IOException("failure tologin", le);
}
if (LOG.isDebugEnabled()) {
LOG.debug("UGIloginUser:"+loginUser);
}
}
关键代码1
private static LoginContext
newLoginContext(String appName, Subject subject,
javax.security.auth.login.Configuration loginConf)
throws LoginException {
// Temporarily switch the thread's ContextClassLoader to match this
// class's classloader, so that we can properly load HadoopLoginModule
// from the JAAS libraries.
Thread t = Thread.currentThread();
ClassLoader oldCCL = t.getContextClassLoader();
t.setContextClassLoader(HadoopLoginModule.class.getClassLoader());
try {
return new LoginContext(appName, subject, null, loginConf);
} finally {
t.setContextClassLoader(oldCCL);
}
}
public void login() throws LoginException {
loginSucceeded = false;
if (subject == null) {
subject = new Subject();
}
try {
// module invoked in doPrivileged
invokePriv(LOGIN_METHOD);
invokePriv(COMMIT_METHOD);//这里执行了HadoopLoginModule类的commit方法
loginSucceeded = true;
} catch (LoginException le) {
try {
invokePriv(ABORT_METHOD);
} catch (LoginException le2) {
throw le;
}
throw le;
}
}
关键代码2
@Override
public boolean commit() throwsLoginException {
if (LOG.isDebugEnabled()) {
LOG.debug("hadoop logincommit");
}
// if we already have a user, we aredone.
if(!subject.getPrincipals(User.class).isEmpty()) {
if (LOG.isDebugEnabled()) {
LOG.debug("using existingsubject:"+subject.getPrincipals());
}
return true;
}
Principal user = null;
// if we are using kerberos, try it out
if(isAuthenticationMethodEnabled(AuthenticationMethod.KERBEROS)) {
user = getCanonicalUser(KerberosPrincipal.class);
if (LOG.isDebugEnabled()) {
LOG.debug("using kerberosuser:"+user);
}
}
//If we don't have a kerberos user andsecurity is disabled, check
//if user is specified in the environmentor properties
if (!isSecurityEnabled() && (user== null)) {
String envUser =System.getenv(HADOOP_USER_NAME);
if (envUser == null) {
envUser =System.getProperty(HADOOP_USER_NAME);//从环境变量HADOOP_USER_NAME中获取用户
}
user = envUser == null ? null : newUser(envUser);
}
// use the OS user
if (user == null) {
user =getCanonicalUser(OS_PRINCIPAL_CLASS);//如果用户还为null,就使用操作系统的用户名
if (LOG.isDebugEnabled()) {
LOG.debug("using localuser:"+user);
}
}
// if we found the user, add ourprincipal
if (user != null) {
if (LOG.isDebugEnabled()) {
LOG.debug("Using user:\"" + user + "\" with name " + user.getName());
}
User userEntry = null;
try {
userEntry = new User(user.getName());
} catch (Exception e) {
throw (LoginException)(newLoginException(e.toString()).initCause(e));
}
if (LOG.isDebugEnabled()) {
LOG.debug("User entry:\"" + userEntry.toString() + "\"" );
}
subject.getPrincipals().add(userEntry);
return true;
}
LOG.error("Can't find user in "+ subject);
throw new LoginException("Can't finduser name");
}
所以,如果想在Java使用Hadoop API操作hdfs时,直接设置
System.setProperty("HADOOP_USER_NAME","user");
即可
public void login() throws LoginException {
loginSucceeded = false;
if (subject == null) {
subject = new Subject();
}
try {
// module invoked in doPrivileged
invokePriv(LOGIN_METHOD);
invokePriv(COMMIT_METHOD);//这里执行了HadoopLoginModule类的commit方法
loginSucceeded = true;
} catch (LoginException le) {
try {
invokePriv(ABORT_METHOD);
} catch (LoginException le2) {
throw le;
}
throw le;
}
}
关键代码2
@Override
public boolean commit() throwsLoginException {
if (LOG.isDebugEnabled()) {
LOG.debug("hadoop logincommit");
}
// if we already have a user, we aredone.
if(!subject.getPrincipals(User.class).isEmpty()) {
if (LOG.isDebugEnabled()) {
LOG.debug("using existingsubject:"+subject.getPrincipals());
}
return true;
}
Principal user = null;
// if we are using kerberos, try it out
if(isAuthenticationMethodEnabled(AuthenticationMethod.KERBEROS)) {
user = getCanonicalUser(KerberosPrincipal.class);
if (LOG.isDebugEnabled()) {
LOG.debug("using kerberosuser:"+user);
}
}
//If we don't have a kerberos user andsecurity is disabled, check
//if user is specified in the environmentor properties
if (!isSecurityEnabled() && (user== null)) {
String envUser =System.getenv(HADOOP_USER_NAME);
if (envUser == null) {
envUser =System.getProperty(HADOOP_USER_NAME);//从环境变量HADOOP_USER_NAME中获取用户
}
user = envUser == null ? null : newUser(envUser);
}
// use the OS user
if (user == null) {
user =getCanonicalUser(OS_PRINCIPAL_CLASS);//如果用户还为null,就使用操作系统的用户名
if (LOG.isDebugEnabled()) {
LOG.debug("using localuser:"+user);
}
}
// if we found the user, add ourprincipal
if (user != null) {
if (LOG.isDebugEnabled()) {
LOG.debug("Using user:\"" + user + "\" with name " + user.getName());
}
User userEntry = null;
try {
userEntry = new User(user.getName());
} catch (Exception e) {
throw (LoginException)(newLoginException(e.toString()).initCause(e));
}
if (LOG.isDebugEnabled()) {
LOG.debug("User entry:\"" + userEntry.toString() + "\"" );
}
subject.getPrincipals().add(userEntry);
return true;
}
LOG.error("Can't find user in "+ subject);
throw new LoginException("Can't finduser name");
}
后来有了新的api,在创建FileSystem的时候,直接传入用户参数
public static FileSystem newInstance(final URI uri, final Configuration conf,
final String user) throws IOException, InterruptedException {
String ticketCachePath =
conf.get(CommonConfigurationKeys.KERBEROS_TICKET_CACHE_PATH);
UserGroupInformation ugi =
UserGroupInformation.getBestUGI(ticketCachePath, user);
return ugi.doAs(new PrivilegedExceptionAction<FileSystem>() {
@Override
public FileSystem run() throws IOException {
return newInstance(uri,conf);
}
});
}