001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.security; 019 020import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN; 021import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN_DEFAULT; 022import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_USER_GROUP_METRICS_PERCENTILES_INTERVALS; 023import static org.apache.hadoop.security.UGIExceptionMessages.*; 024import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_TREAT_SUBJECT_EXTERNAL_KEY; 025import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_TREAT_SUBJECT_EXTERNAL_DEFAULT; 026import static org.apache.hadoop.util.PlatformName.IBM_JAVA; 027 028import com.google.common.annotations.VisibleForTesting; 029 030import java.io.File; 031import java.io.IOException; 032import java.lang.reflect.UndeclaredThrowableException; 033import java.security.AccessControlContext; 034import java.security.AccessController; 035import java.security.Principal; 036import java.security.PrivilegedAction; 037import java.security.PrivilegedActionException; 038import java.security.PrivilegedExceptionAction; 039import java.util.ArrayList; 040import java.util.Arrays; 041import java.util.Collection; 042import java.util.Collections; 043import java.util.HashMap; 044import java.util.Iterator; 045import java.util.List; 046import java.util.Map; 047import java.util.Set; 048import java.util.concurrent.TimeUnit; 049 050import javax.security.auth.DestroyFailedException; 051import javax.security.auth.Subject; 052import javax.security.auth.callback.CallbackHandler; 053import javax.security.auth.kerberos.KerberosPrincipal; 054import javax.security.auth.kerberos.KerberosTicket; 055import javax.security.auth.kerberos.KeyTab; 056import javax.security.auth.login.AppConfigurationEntry; 057import javax.security.auth.login.AppConfigurationEntry.LoginModuleControlFlag; 058import javax.security.auth.login.LoginContext; 059import javax.security.auth.login.LoginException; 060import javax.security.auth.spi.LoginModule; 061 062import org.apache.hadoop.io.retry.RetryPolicies; 063import org.apache.hadoop.classification.InterfaceAudience; 064import org.apache.hadoop.classification.InterfaceStability; 065import org.apache.hadoop.conf.Configuration; 066import org.apache.hadoop.io.Text; 067import org.apache.hadoop.io.retry.RetryPolicy; 068import org.apache.hadoop.metrics2.annotation.Metric; 069import org.apache.hadoop.metrics2.annotation.Metrics; 070import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; 071import org.apache.hadoop.metrics2.lib.MetricsRegistry; 072import org.apache.hadoop.metrics2.lib.MutableGaugeInt; 073import org.apache.hadoop.metrics2.lib.MutableGaugeLong; 074import org.apache.hadoop.metrics2.lib.MutableQuantiles; 075import org.apache.hadoop.metrics2.lib.MutableRate; 076import org.apache.hadoop.security.SaslRpcServer.AuthMethod; 077import org.apache.hadoop.security.authentication.util.KerberosUtil; 078import org.apache.hadoop.security.token.Token; 079import org.apache.hadoop.security.token.TokenIdentifier; 080import org.apache.hadoop.util.Shell; 081import org.apache.hadoop.util.StringUtils; 082import org.apache.hadoop.util.Time; 083 084import org.slf4j.Logger; 085import org.slf4j.LoggerFactory; 086 087/** 088 * User and group information for Hadoop. 089 * This class wraps around a JAAS Subject and provides methods to determine the 090 * user's username and groups. It supports both the Windows, Unix and Kerberos 091 * login modules. 092 */ 093@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce", "HBase", "Hive", "Oozie"}) 094@InterfaceStability.Evolving 095public class UserGroupInformation { 096 @VisibleForTesting 097 static final Logger LOG = LoggerFactory.getLogger( 098 UserGroupInformation.class); 099 100 /** 101 * Percentage of the ticket window to use before we renew ticket. 102 */ 103 private static final float TICKET_RENEW_WINDOW = 0.80f; 104 private static boolean shouldRenewImmediatelyForTests = false; 105 static final String HADOOP_USER_NAME = "HADOOP_USER_NAME"; 106 static final String HADOOP_PROXY_USER = "HADOOP_PROXY_USER"; 107 108 /** 109 * For the purposes of unit tests, we want to test login 110 * from keytab and don't want to wait until the renew 111 * window (controlled by TICKET_RENEW_WINDOW). 112 * @param immediate true if we should login without waiting for ticket window 113 */ 114 @VisibleForTesting 115 public static void setShouldRenewImmediatelyForTests(boolean immediate) { 116 shouldRenewImmediatelyForTests = immediate; 117 } 118 119 /** 120 * UgiMetrics maintains UGI activity statistics 121 * and publishes them through the metrics interfaces. 122 */ 123 @Metrics(about="User and group related metrics", context="ugi") 124 static class UgiMetrics { 125 final MetricsRegistry registry = new MetricsRegistry("UgiMetrics"); 126 127 @Metric("Rate of successful kerberos logins and latency (milliseconds)") 128 MutableRate loginSuccess; 129 @Metric("Rate of failed kerberos logins and latency (milliseconds)") 130 MutableRate loginFailure; 131 @Metric("GetGroups") MutableRate getGroups; 132 MutableQuantiles[] getGroupsQuantiles; 133 @Metric("Renewal failures since startup") 134 private MutableGaugeLong renewalFailuresTotal; 135 @Metric("Renewal failures since last successful login") 136 private MutableGaugeInt renewalFailures; 137 138 static UgiMetrics create() { 139 return DefaultMetricsSystem.instance().register(new UgiMetrics()); 140 } 141 142 void addGetGroups(long latency) { 143 getGroups.add(latency); 144 if (getGroupsQuantiles != null) { 145 for (MutableQuantiles q : getGroupsQuantiles) { 146 q.add(latency); 147 } 148 } 149 } 150 151 MutableGaugeInt getRenewalFailures() { 152 return renewalFailures; 153 } 154 } 155 156 /** 157 * A login module that looks at the Kerberos, Unix, or Windows principal and 158 * adds the corresponding UserName. 159 */ 160 @InterfaceAudience.Private 161 public static class HadoopLoginModule implements LoginModule { 162 private Subject subject; 163 164 @Override 165 public boolean abort() throws LoginException { 166 return true; 167 } 168 169 private <T extends Principal> T getCanonicalUser(Class<T> cls) { 170 for(T user: subject.getPrincipals(cls)) { 171 return user; 172 } 173 return null; 174 } 175 176 @Override 177 public boolean commit() throws LoginException { 178 if (LOG.isDebugEnabled()) { 179 LOG.debug("hadoop login commit"); 180 } 181 // if we already have a user, we are done. 182 if (!subject.getPrincipals(User.class).isEmpty()) { 183 if (LOG.isDebugEnabled()) { 184 LOG.debug("using existing subject:"+subject.getPrincipals()); 185 } 186 return true; 187 } 188 Principal user = null; 189 // if we are using kerberos, try it out 190 if (isAuthenticationMethodEnabled(AuthenticationMethod.KERBEROS)) { 191 user = getCanonicalUser(KerberosPrincipal.class); 192 if (LOG.isDebugEnabled()) { 193 LOG.debug("using kerberos user:"+user); 194 } 195 } 196 //If we don't have a kerberos user and security is disabled, check 197 //if user is specified in the environment or properties 198 if (!isSecurityEnabled() && (user == null)) { 199 String envUser = System.getenv(HADOOP_USER_NAME); 200 if (envUser == null) { 201 envUser = System.getProperty(HADOOP_USER_NAME); 202 } 203 user = envUser == null ? null : new User(envUser); 204 } 205 // use the OS user 206 if (user == null) { 207 user = getCanonicalUser(OS_PRINCIPAL_CLASS); 208 if (LOG.isDebugEnabled()) { 209 LOG.debug("using local user:"+user); 210 } 211 } 212 // if we found the user, add our principal 213 if (user != null) { 214 if (LOG.isDebugEnabled()) { 215 LOG.debug("Using user: \"" + user + "\" with name " + user.getName()); 216 } 217 218 User userEntry = null; 219 try { 220 userEntry = new User(user.getName()); 221 } catch (Exception e) { 222 throw (LoginException)(new LoginException(e.toString()).initCause(e)); 223 } 224 if (LOG.isDebugEnabled()) { 225 LOG.debug("User entry: \"" + userEntry.toString() + "\"" ); 226 } 227 228 subject.getPrincipals().add(userEntry); 229 return true; 230 } 231 LOG.error("Can't find user in " + subject); 232 throw new LoginException("Can't find user name"); 233 } 234 235 @Override 236 public void initialize(Subject subject, CallbackHandler callbackHandler, 237 Map<String, ?> sharedState, Map<String, ?> options) { 238 this.subject = subject; 239 } 240 241 @Override 242 public boolean login() throws LoginException { 243 if (LOG.isDebugEnabled()) { 244 LOG.debug("hadoop login"); 245 } 246 return true; 247 } 248 249 @Override 250 public boolean logout() throws LoginException { 251 if (LOG.isDebugEnabled()) { 252 LOG.debug("hadoop logout"); 253 } 254 return true; 255 } 256 } 257 258 /** Metrics to track UGI activity */ 259 static UgiMetrics metrics = UgiMetrics.create(); 260 /** The auth method to use */ 261 private static AuthenticationMethod authenticationMethod; 262 /** Server-side groups fetching service */ 263 private static Groups groups; 264 /** Min time (in seconds) before relogin for Kerberos */ 265 private static long kerberosMinSecondsBeforeRelogin; 266 /** The configuration to use */ 267 268 /* 269 * This config is a temporary one for backward compatibility. 270 * It means whether to treat the subject passed to 271 * UserGroupInformation(Subject) as external. If true, 272 * - no renewal thread will be created to do the renew credential 273 * - reloginFromKeytab() and reloginFromTicketCache will not renew 274 * credential. 275 * and it assumes that the owner of the subject to renew; if false, it means 276 * to retain the old behavior prior to fixing HADOOP-13558 and HADOOP-13805. 277 * The default is false. 278 */ 279 private static boolean treatSubjectExternal = false; 280 281 /* 282 * Some test need the renewal thread to be created even if it does 283 * UserGroupInformation.loginUserFromSubject(subject); 284 * The test code may set this variable to true via 285 * setEnableRenewThreadCreationForTest(boolean) 286 * method. 287 */ 288 private static boolean enableRenewThreadCreationForTest = false; 289 290 private static Configuration conf; 291 292 /**Environment variable pointing to the token cache file*/ 293 public static final String HADOOP_TOKEN_FILE_LOCATION = 294 "HADOOP_TOKEN_FILE_LOCATION"; 295 296 /** 297 * A method to initialize the fields that depend on a configuration. 298 * Must be called before useKerberos or groups is used. 299 */ 300 private static void ensureInitialized() { 301 if (conf == null) { 302 synchronized(UserGroupInformation.class) { 303 if (conf == null) { // someone might have beat us 304 initialize(new Configuration(), false); 305 } 306 } 307 } 308 } 309 310 /** 311 * Initialize UGI and related classes. 312 * @param conf the configuration to use 313 */ 314 private static synchronized void initialize(Configuration conf, 315 boolean overrideNameRules) { 316 authenticationMethod = SecurityUtil.getAuthenticationMethod(conf); 317 if (overrideNameRules || !HadoopKerberosName.hasRulesBeenSet()) { 318 try { 319 HadoopKerberosName.setConfiguration(conf); 320 } catch (IOException ioe) { 321 throw new RuntimeException( 322 "Problem with Kerberos auth_to_local name configuration", ioe); 323 } 324 } 325 try { 326 kerberosMinSecondsBeforeRelogin = 1000L * conf.getLong( 327 HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN, 328 HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN_DEFAULT); 329 } 330 catch(NumberFormatException nfe) { 331 throw new IllegalArgumentException("Invalid attribute value for " + 332 HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN + " of " + 333 conf.get(HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN)); 334 } 335 // If we haven't set up testing groups, use the configuration to find it 336 if (!(groups instanceof TestingGroups)) { 337 groups = Groups.getUserToGroupsMappingService(conf); 338 } 339 UserGroupInformation.conf = conf; 340 341 if (metrics.getGroupsQuantiles == null) { 342 int[] intervals = conf.getInts(HADOOP_USER_GROUP_METRICS_PERCENTILES_INTERVALS); 343 if (intervals != null && intervals.length > 0) { 344 final int length = intervals.length; 345 MutableQuantiles[] getGroupsQuantiles = new MutableQuantiles[length]; 346 for (int i = 0; i < length; i++) { 347 getGroupsQuantiles[i] = metrics.registry.newQuantiles( 348 "getGroups" + intervals[i] + "s", 349 "Get groups", "ops", "latency", intervals[i]); 350 } 351 metrics.getGroupsQuantiles = getGroupsQuantiles; 352 } 353 } 354 355 treatSubjectExternal = conf.getBoolean(HADOOP_TREAT_SUBJECT_EXTERNAL_KEY, 356 HADOOP_TREAT_SUBJECT_EXTERNAL_DEFAULT); 357 if (treatSubjectExternal) { 358 LOG.info("Config " + HADOOP_TREAT_SUBJECT_EXTERNAL_KEY + " is set to " 359 + "true, the owner of the subject passed to " 360 + " UserGroupInformation(Subject) is supposed to renew the " 361 + "credential."); 362 } 363 } 364 365 /** 366 * Set the static configuration for UGI. 367 * In particular, set the security authentication mechanism and the 368 * group look up service. 369 * @param conf the configuration to use 370 */ 371 @InterfaceAudience.Public 372 @InterfaceStability.Evolving 373 public static void setConfiguration(Configuration conf) { 374 initialize(conf, true); 375 } 376 377 @InterfaceAudience.Private 378 @VisibleForTesting 379 static void setEnableRenewThreadCreationForTest(boolean b) { 380 enableRenewThreadCreationForTest = b; 381 } 382 383 @InterfaceAudience.Private 384 @VisibleForTesting 385 static boolean getEnableRenewThreadCreationForTest() { 386 return enableRenewThreadCreationForTest; 387 } 388 389 @InterfaceAudience.Private 390 @VisibleForTesting 391 public static void reset() { 392 authenticationMethod = null; 393 conf = null; 394 groups = null; 395 setLoginUser(null); 396 HadoopKerberosName.setRules(null); 397 setEnableRenewThreadCreationForTest(false); 398 } 399 400 /** 401 * Determine if UserGroupInformation is using Kerberos to determine 402 * user identities or is relying on simple authentication 403 * 404 * @return true if UGI is working in a secure environment 405 */ 406 public static boolean isSecurityEnabled() { 407 return !isAuthenticationMethodEnabled(AuthenticationMethod.SIMPLE); 408 } 409 410 @InterfaceAudience.Private 411 @InterfaceStability.Evolving 412 private static boolean isAuthenticationMethodEnabled(AuthenticationMethod method) { 413 ensureInitialized(); 414 return (authenticationMethod == method); 415 } 416 417 /** 418 * Information about the logged in user. 419 */ 420 private static UserGroupInformation loginUser = null; 421 private static String keytabPrincipal = null; 422 private static String keytabFile = null; 423 424 private final Subject subject; 425 // All non-static fields must be read-only caches that come from the subject. 426 private final User user; 427 private final boolean isKeytab; 428 private final boolean isKrbTkt; 429 private final boolean isLoginExternal; 430 431 private static String OS_LOGIN_MODULE_NAME; 432 private static Class<? extends Principal> OS_PRINCIPAL_CLASS; 433 434 private static final boolean windows = 435 System.getProperty("os.name").startsWith("Windows"); 436 private static final boolean is64Bit = 437 System.getProperty("os.arch").contains("64"); 438 private static final boolean aix = System.getProperty("os.name").equals("AIX"); 439 440 /* Return the OS login module class name */ 441 private static String getOSLoginModuleName() { 442 if (IBM_JAVA) { 443 if (windows) { 444 return is64Bit ? "com.ibm.security.auth.module.Win64LoginModule" 445 : "com.ibm.security.auth.module.NTLoginModule"; 446 } else if (aix) { 447 return is64Bit ? "com.ibm.security.auth.module.AIX64LoginModule" 448 : "com.ibm.security.auth.module.AIXLoginModule"; 449 } else { 450 return "com.ibm.security.auth.module.LinuxLoginModule"; 451 } 452 } else { 453 return windows ? "com.sun.security.auth.module.NTLoginModule" 454 : "com.sun.security.auth.module.UnixLoginModule"; 455 } 456 } 457 458 /* Return the OS principal class */ 459 @SuppressWarnings("unchecked") 460 private static Class<? extends Principal> getOsPrincipalClass() { 461 ClassLoader cl = ClassLoader.getSystemClassLoader(); 462 try { 463 String principalClass = null; 464 if (IBM_JAVA) { 465 if (is64Bit) { 466 principalClass = "com.ibm.security.auth.UsernamePrincipal"; 467 } else { 468 if (windows) { 469 principalClass = "com.ibm.security.auth.NTUserPrincipal"; 470 } else if (aix) { 471 principalClass = "com.ibm.security.auth.AIXPrincipal"; 472 } else { 473 principalClass = "com.ibm.security.auth.LinuxPrincipal"; 474 } 475 } 476 } else { 477 principalClass = windows ? "com.sun.security.auth.NTUserPrincipal" 478 : "com.sun.security.auth.UnixPrincipal"; 479 } 480 return (Class<? extends Principal>) cl.loadClass(principalClass); 481 } catch (ClassNotFoundException e) { 482 LOG.error("Unable to find JAAS classes:" + e.getMessage()); 483 } 484 return null; 485 } 486 static { 487 OS_LOGIN_MODULE_NAME = getOSLoginModuleName(); 488 OS_PRINCIPAL_CLASS = getOsPrincipalClass(); 489 } 490 491 private static class RealUser implements Principal { 492 private final UserGroupInformation realUser; 493 494 RealUser(UserGroupInformation realUser) { 495 this.realUser = realUser; 496 } 497 498 @Override 499 public String getName() { 500 return realUser.getUserName(); 501 } 502 503 public UserGroupInformation getRealUser() { 504 return realUser; 505 } 506 507 @Override 508 public boolean equals(Object o) { 509 if (this == o) { 510 return true; 511 } else if (o == null || getClass() != o.getClass()) { 512 return false; 513 } else { 514 return realUser.equals(((RealUser) o).realUser); 515 } 516 } 517 518 @Override 519 public int hashCode() { 520 return realUser.hashCode(); 521 } 522 523 @Override 524 public String toString() { 525 return realUser.toString(); 526 } 527 } 528 529 /** 530 * A JAAS configuration that defines the login modules that we want 531 * to use for login. 532 */ 533 private static class HadoopConfiguration 534 extends javax.security.auth.login.Configuration { 535 private static final String SIMPLE_CONFIG_NAME = "hadoop-simple"; 536 private static final String USER_KERBEROS_CONFIG_NAME = 537 "hadoop-user-kerberos"; 538 private static final String KEYTAB_KERBEROS_CONFIG_NAME = 539 "hadoop-keytab-kerberos"; 540 541 private static final Map<String, String> BASIC_JAAS_OPTIONS = 542 new HashMap<String,String>(); 543 static { 544 String jaasEnvVar = System.getenv("HADOOP_JAAS_DEBUG"); 545 if (jaasEnvVar != null && "true".equalsIgnoreCase(jaasEnvVar)) { 546 BASIC_JAAS_OPTIONS.put("debug", "true"); 547 } 548 } 549 550 private static final AppConfigurationEntry OS_SPECIFIC_LOGIN = 551 new AppConfigurationEntry(OS_LOGIN_MODULE_NAME, 552 LoginModuleControlFlag.REQUIRED, 553 BASIC_JAAS_OPTIONS); 554 private static final AppConfigurationEntry HADOOP_LOGIN = 555 new AppConfigurationEntry(HadoopLoginModule.class.getName(), 556 LoginModuleControlFlag.REQUIRED, 557 BASIC_JAAS_OPTIONS); 558 private static final Map<String,String> USER_KERBEROS_OPTIONS = 559 new HashMap<String,String>(); 560 static { 561 if (IBM_JAVA) { 562 USER_KERBEROS_OPTIONS.put("useDefaultCcache", "true"); 563 } else { 564 USER_KERBEROS_OPTIONS.put("doNotPrompt", "true"); 565 USER_KERBEROS_OPTIONS.put("useTicketCache", "true"); 566 } 567 String ticketCache = System.getenv("KRB5CCNAME"); 568 if (ticketCache != null) { 569 if (IBM_JAVA) { 570 // The first value searched when "useDefaultCcache" is used. 571 System.setProperty("KRB5CCNAME", ticketCache); 572 } else { 573 USER_KERBEROS_OPTIONS.put("ticketCache", ticketCache); 574 } 575 } 576 USER_KERBEROS_OPTIONS.put("renewTGT", "true"); 577 USER_KERBEROS_OPTIONS.putAll(BASIC_JAAS_OPTIONS); 578 } 579 private static final AppConfigurationEntry USER_KERBEROS_LOGIN = 580 new AppConfigurationEntry(KerberosUtil.getKrb5LoginModuleName(), 581 LoginModuleControlFlag.OPTIONAL, 582 USER_KERBEROS_OPTIONS); 583 private static final Map<String,String> KEYTAB_KERBEROS_OPTIONS = 584 new HashMap<String,String>(); 585 static { 586 if (IBM_JAVA) { 587 KEYTAB_KERBEROS_OPTIONS.put("credsType", "both"); 588 } else { 589 KEYTAB_KERBEROS_OPTIONS.put("doNotPrompt", "true"); 590 KEYTAB_KERBEROS_OPTIONS.put("useKeyTab", "true"); 591 KEYTAB_KERBEROS_OPTIONS.put("storeKey", "true"); 592 } 593 KEYTAB_KERBEROS_OPTIONS.put("refreshKrb5Config", "true"); 594 KEYTAB_KERBEROS_OPTIONS.putAll(BASIC_JAAS_OPTIONS); 595 } 596 private static final AppConfigurationEntry KEYTAB_KERBEROS_LOGIN = 597 new AppConfigurationEntry(KerberosUtil.getKrb5LoginModuleName(), 598 LoginModuleControlFlag.REQUIRED, 599 KEYTAB_KERBEROS_OPTIONS); 600 601 private static final AppConfigurationEntry[] SIMPLE_CONF = 602 new AppConfigurationEntry[]{OS_SPECIFIC_LOGIN, HADOOP_LOGIN}; 603 604 private static final AppConfigurationEntry[] USER_KERBEROS_CONF = 605 new AppConfigurationEntry[]{OS_SPECIFIC_LOGIN, USER_KERBEROS_LOGIN, 606 HADOOP_LOGIN}; 607 608 private static final AppConfigurationEntry[] KEYTAB_KERBEROS_CONF = 609 new AppConfigurationEntry[]{KEYTAB_KERBEROS_LOGIN, HADOOP_LOGIN}; 610 611 @Override 612 public AppConfigurationEntry[] getAppConfigurationEntry(String appName) { 613 if (SIMPLE_CONFIG_NAME.equals(appName)) { 614 return SIMPLE_CONF; 615 } else if (USER_KERBEROS_CONFIG_NAME.equals(appName)) { 616 return USER_KERBEROS_CONF; 617 } else if (KEYTAB_KERBEROS_CONFIG_NAME.equals(appName)) { 618 if (IBM_JAVA) { 619 KEYTAB_KERBEROS_OPTIONS.put("useKeytab", 620 prependFileAuthority(keytabFile)); 621 } else { 622 KEYTAB_KERBEROS_OPTIONS.put("keyTab", keytabFile); 623 } 624 KEYTAB_KERBEROS_OPTIONS.put("principal", keytabPrincipal); 625 return KEYTAB_KERBEROS_CONF; 626 } 627 return null; 628 } 629 } 630 631 private static String prependFileAuthority(String keytabPath) { 632 return keytabPath.startsWith("file://") ? keytabPath 633 : "file://" + keytabPath; 634 } 635 636 /** 637 * Represents a javax.security configuration that is created at runtime. 638 */ 639 private static class DynamicConfiguration 640 extends javax.security.auth.login.Configuration { 641 private AppConfigurationEntry[] ace; 642 643 DynamicConfiguration(AppConfigurationEntry[] ace) { 644 this.ace = ace; 645 } 646 647 @Override 648 public AppConfigurationEntry[] getAppConfigurationEntry(String appName) { 649 return ace; 650 } 651 } 652 653 private static LoginContext 654 newLoginContext(String appName, Subject subject, 655 javax.security.auth.login.Configuration loginConf) 656 throws LoginException { 657 // Temporarily switch the thread's ContextClassLoader to match this 658 // class's classloader, so that we can properly load HadoopLoginModule 659 // from the JAAS libraries. 660 Thread t = Thread.currentThread(); 661 ClassLoader oldCCL = t.getContextClassLoader(); 662 t.setContextClassLoader(HadoopLoginModule.class.getClassLoader()); 663 try { 664 return new LoginContext(appName, subject, null, loginConf); 665 } finally { 666 t.setContextClassLoader(oldCCL); 667 } 668 } 669 670 private LoginContext getLogin() { 671 return user.getLogin(); 672 } 673 674 private void setLogin(LoginContext login) { 675 user.setLogin(login); 676 } 677 678 /** 679 * Create a UserGroupInformation for the given subject. 680 * This does not change the subject or acquire new credentials. 681 * 682 * The creator of subject is responsible for renewing credentials. 683 * @param subject the user's subject 684 */ 685 UserGroupInformation(Subject subject) { 686 this(subject, treatSubjectExternal); 687 } 688 689 /** 690 * Create a UGI from the given subject. 691 * @param subject the subject 692 * @param isLoginExternal if the subject's keytab is managed by other UGI. 693 * Setting this to true will prevent UGI from attempting 694 * to login the keytab, or to renew it. 695 */ 696 private UserGroupInformation(Subject subject, final boolean isLoginExternal) { 697 this.subject = subject; 698 this.user = subject.getPrincipals(User.class).iterator().next(); 699 this.isKeytab = !subject.getPrivateCredentials(KeyTab.class).isEmpty(); 700 this.isKrbTkt = !subject.getPrivateCredentials(KerberosTicket.class).isEmpty(); 701 this.isLoginExternal = isLoginExternal; 702 } 703 704 /** 705 * checks if logged in using kerberos 706 * @return true if the subject logged via keytab or has a Kerberos TGT 707 */ 708 public boolean hasKerberosCredentials() { 709 return isKeytab || isKrbTkt; 710 } 711 712 /** 713 * Return the current user, including any doAs in the current stack. 714 * @return the current user 715 * @throws IOException if login fails 716 */ 717 @InterfaceAudience.Public 718 @InterfaceStability.Evolving 719 public synchronized 720 static UserGroupInformation getCurrentUser() throws IOException { 721 AccessControlContext context = AccessController.getContext(); 722 Subject subject = Subject.getSubject(context); 723 if (subject == null || subject.getPrincipals(User.class).isEmpty()) { 724 return getLoginUser(); 725 } else { 726 return new UserGroupInformation(subject); 727 } 728 } 729 730 /** 731 * Find the most appropriate UserGroupInformation to use 732 * 733 * @param ticketCachePath The Kerberos ticket cache path, or NULL 734 * if none is specfied 735 * @param user The user name, or NULL if none is specified. 736 * 737 * @return The most appropriate UserGroupInformation 738 */ 739 public static UserGroupInformation getBestUGI( 740 String ticketCachePath, String user) throws IOException { 741 if (ticketCachePath != null) { 742 return getUGIFromTicketCache(ticketCachePath, user); 743 } else if (user == null) { 744 return getCurrentUser(); 745 } else { 746 return createRemoteUser(user); 747 } 748 } 749 750 /** 751 * Create a UserGroupInformation from a Kerberos ticket cache. 752 * 753 * @param user The principal name to load from the ticket 754 * cache 755 * @param ticketCachePath the path to the ticket cache file 756 * 757 * @throws IOException if the kerberos login fails 758 */ 759 @InterfaceAudience.Public 760 @InterfaceStability.Evolving 761 public static UserGroupInformation getUGIFromTicketCache( 762 String ticketCache, String user) throws IOException { 763 if (!isAuthenticationMethodEnabled(AuthenticationMethod.KERBEROS)) { 764 return getBestUGI(null, user); 765 } 766 try { 767 Map<String,String> krbOptions = new HashMap<String,String>(); 768 if (IBM_JAVA) { 769 krbOptions.put("useDefaultCcache", "true"); 770 // The first value searched when "useDefaultCcache" is used. 771 System.setProperty("KRB5CCNAME", ticketCache); 772 } else { 773 krbOptions.put("doNotPrompt", "true"); 774 krbOptions.put("useTicketCache", "true"); 775 krbOptions.put("useKeyTab", "false"); 776 krbOptions.put("ticketCache", ticketCache); 777 } 778 krbOptions.put("renewTGT", "false"); 779 krbOptions.putAll(HadoopConfiguration.BASIC_JAAS_OPTIONS); 780 AppConfigurationEntry ace = new AppConfigurationEntry( 781 KerberosUtil.getKrb5LoginModuleName(), 782 LoginModuleControlFlag.REQUIRED, 783 krbOptions); 784 DynamicConfiguration dynConf = 785 new DynamicConfiguration(new AppConfigurationEntry[]{ ace }); 786 LoginContext login = newLoginContext( 787 HadoopConfiguration.USER_KERBEROS_CONFIG_NAME, null, dynConf); 788 login.login(); 789 790 Subject loginSubject = login.getSubject(); 791 Set<Principal> loginPrincipals = loginSubject.getPrincipals(); 792 if (loginPrincipals.isEmpty()) { 793 throw new RuntimeException("No login principals found!"); 794 } 795 if (loginPrincipals.size() != 1) { 796 LOG.warn("found more than one principal in the ticket cache file " + 797 ticketCache); 798 } 799 User ugiUser = new User(loginPrincipals.iterator().next().getName(), 800 AuthenticationMethod.KERBEROS, login); 801 loginSubject.getPrincipals().add(ugiUser); 802 UserGroupInformation ugi = new UserGroupInformation(loginSubject, false); 803 ugi.setLogin(login); 804 ugi.setAuthenticationMethod(AuthenticationMethod.KERBEROS); 805 return ugi; 806 } catch (LoginException le) { 807 KerberosAuthException kae = 808 new KerberosAuthException(FAILURE_TO_LOGIN, le); 809 kae.setUser(user); 810 kae.setTicketCacheFile(ticketCache); 811 throw kae; 812 } 813 } 814 815 /** 816 * Create a UserGroupInformation from a Subject with Kerberos principal. 817 * 818 * @param subject The KerberosPrincipal to use in UGI. 819 * The creator of subject is responsible for 820 * renewing credentials. 821 * 822 * @throws IOException 823 * @throws KerberosAuthException if the kerberos login fails 824 */ 825 public static UserGroupInformation getUGIFromSubject(Subject subject) 826 throws IOException { 827 if (subject == null) { 828 throw new KerberosAuthException(SUBJECT_MUST_NOT_BE_NULL); 829 } 830 831 if (subject.getPrincipals(KerberosPrincipal.class).isEmpty()) { 832 throw new KerberosAuthException(SUBJECT_MUST_CONTAIN_PRINCIPAL); 833 } 834 835 KerberosPrincipal principal = 836 subject.getPrincipals(KerberosPrincipal.class).iterator().next(); 837 838 User ugiUser = new User(principal.getName(), 839 AuthenticationMethod.KERBEROS, null); 840 subject.getPrincipals().add(ugiUser); 841 UserGroupInformation ugi = new UserGroupInformation(subject); 842 ugi.setLogin(null); 843 ugi.setAuthenticationMethod(AuthenticationMethod.KERBEROS); 844 return ugi; 845 } 846 847 /** 848 * Get the currently logged in user. 849 * @return the logged in user 850 * @throws IOException if login fails 851 */ 852 @InterfaceAudience.Public 853 @InterfaceStability.Evolving 854 public synchronized 855 static UserGroupInformation getLoginUser() throws IOException { 856 if (loginUser == null) { 857 loginUserFromSubject(null); 858 } 859 return loginUser; 860 } 861 862 /** 863 * remove the login method that is followed by a space from the username 864 * e.g. "jack (auth:SIMPLE)" -> "jack" 865 * 866 * @param userName 867 * @return userName without login method 868 */ 869 public static String trimLoginMethod(String userName) { 870 int spaceIndex = userName.indexOf(' '); 871 if (spaceIndex >= 0) { 872 userName = userName.substring(0, spaceIndex); 873 } 874 return userName; 875 } 876 877 /** 878 * Log in a user using the given subject 879 * @parma subject the subject to use when logging in a user, or null to 880 * create a new subject. 881 * 882 * If subject is not null, the creator of subject is responsible for renewing 883 * credentials. 884 * 885 * @throws IOException if login fails 886 */ 887 @InterfaceAudience.Public 888 @InterfaceStability.Evolving 889 public synchronized 890 static void loginUserFromSubject(Subject subject) throws IOException { 891 ensureInitialized(); 892 boolean externalSubject = false; 893 try { 894 if (subject == null) { 895 subject = new Subject(); 896 } else { 897 if (LOG.isDebugEnabled()) { 898 LOG.debug("Treat subject external: " + treatSubjectExternal 899 + ". When true, assuming keytab is managed extenally since " 900 + " logged in from subject"); 901 } 902 externalSubject = treatSubjectExternal; 903 } 904 LoginContext login = 905 newLoginContext(authenticationMethod.getLoginAppName(), 906 subject, new HadoopConfiguration()); 907 login.login(); 908 909 UserGroupInformation realUser = 910 new UserGroupInformation(subject, externalSubject); 911 realUser.setLogin(login); 912 realUser.setAuthenticationMethod(authenticationMethod); 913 // If the HADOOP_PROXY_USER environment variable or property 914 // is specified, create a proxy user as the logged in user. 915 String proxyUser = System.getenv(HADOOP_PROXY_USER); 916 if (proxyUser == null) { 917 proxyUser = System.getProperty(HADOOP_PROXY_USER); 918 } 919 loginUser = proxyUser == null ? realUser : createProxyUser(proxyUser, realUser); 920 921 String fileLocation = System.getenv(HADOOP_TOKEN_FILE_LOCATION); 922 if (fileLocation != null) { 923 // Load the token storage file and put all of the tokens into the 924 // user. Don't use the FileSystem API for reading since it has a lock 925 // cycle (HADOOP-9212). 926 Credentials cred = Credentials.readTokenStorageFile( 927 new File(fileLocation), conf); 928 loginUser.addCredentials(cred); 929 } 930 loginUser.spawnAutoRenewalThreadForUserCreds(); 931 } catch (LoginException le) { 932 LOG.debug("failure to login", le); 933 throw new KerberosAuthException(FAILURE_TO_LOGIN, le); 934 } 935 if (LOG.isDebugEnabled()) { 936 LOG.debug("UGI loginUser:"+loginUser); 937 } 938 } 939 940 @InterfaceAudience.Private 941 @InterfaceStability.Unstable 942 @VisibleForTesting 943 public synchronized static void setLoginUser(UserGroupInformation ugi) { 944 // if this is to become stable, should probably logout the currently 945 // logged in ugi if it's different 946 loginUser = ugi; 947 } 948 949 /** 950 * Is this user logged in from a keytab file? 951 * @return true if the credentials are from a keytab file. 952 */ 953 public boolean isFromKeytab() { 954 return isKeytab; 955 } 956 957 /** 958 * Get the Kerberos TGT 959 * @return the user's TGT or null if none was found 960 */ 961 private synchronized KerberosTicket getTGT() { 962 Set<KerberosTicket> tickets = subject 963 .getPrivateCredentials(KerberosTicket.class); 964 for (KerberosTicket ticket : tickets) { 965 if (SecurityUtil.isOriginalTGT(ticket)) { 966 return ticket; 967 } 968 } 969 return null; 970 } 971 972 private long getRefreshTime(KerberosTicket tgt) { 973 long start = tgt.getStartTime().getTime(); 974 long end = tgt.getEndTime().getTime(); 975 return start + (long) ((end - start) * TICKET_RENEW_WINDOW); 976 } 977 978 /** 979 * Should relogin if security is enabled using Kerberos, and 980 * the Subject is not owned by another UGI. 981 * @return true if this UGI should relogin 982 */ 983 private boolean shouldRelogin() { 984 return isSecurityEnabled() 985 && user.getAuthenticationMethod() == AuthenticationMethod.KERBEROS 986 && !isLoginExternal; 987 } 988 989 /**Spawn a thread to do periodic renewals of kerberos credentials*/ 990 private void spawnAutoRenewalThreadForUserCreds() { 991 if (getEnableRenewThreadCreationForTest()) { 992 LOG.warn("Spawning thread to auto renew user credential since " + 993 " enableRenewThreadCreationForTest was set to true."); 994 } else if (!shouldRelogin() || isKeytab) { 995 return; 996 } 997 998 //spawn thread only if we have kerb credentials 999 Thread t = new Thread(new Runnable() { 1000 1001 @Override 1002 public void run() { 1003 String cmd = conf.get("hadoop.kerberos.kinit.command", "kinit"); 1004 KerberosTicket tgt = getTGT(); 1005 if (tgt == null) { 1006 return; 1007 } 1008 long nextRefresh = getRefreshTime(tgt); 1009 RetryPolicy rp = null; 1010 while (true) { 1011 try { 1012 long now = Time.now(); 1013 if (LOG.isDebugEnabled()) { 1014 LOG.debug("Current time is " + now); 1015 LOG.debug("Next refresh is " + nextRefresh); 1016 } 1017 if (now < nextRefresh) { 1018 Thread.sleep(nextRefresh - now); 1019 } 1020 Shell.execCommand(cmd, "-R"); 1021 if (LOG.isDebugEnabled()) { 1022 LOG.debug("renewed ticket"); 1023 } 1024 reloginFromTicketCache(); 1025 tgt = getTGT(); 1026 if (tgt == null) { 1027 LOG.warn("No TGT after renewal. Aborting renew thread for " + 1028 getUserName()); 1029 return; 1030 } 1031 nextRefresh = Math.max(getRefreshTime(tgt), 1032 now + kerberosMinSecondsBeforeRelogin); 1033 metrics.renewalFailures.set(0); 1034 rp = null; 1035 } catch (InterruptedException ie) { 1036 LOG.warn("Terminating renewal thread"); 1037 return; 1038 } catch (IOException ie) { 1039 metrics.renewalFailuresTotal.incr(); 1040 final long tgtEndTime = tgt.getEndTime().getTime(); 1041 LOG.warn("Exception encountered while running the renewal " 1042 + "command for {}. (TGT end time:{}, renewalFailures: {}," 1043 + "renewalFailuresTotal: {})", getUserName(), tgtEndTime, 1044 metrics.renewalFailures, metrics.renewalFailuresTotal, ie); 1045 final long now = Time.now(); 1046 if (rp == null) { 1047 // Use a dummy maxRetries to create the policy. The policy will 1048 // only be used to get next retry time with exponential back-off. 1049 // The final retry time will be later limited within the 1050 // tgt endTime in getNextTgtRenewalTime. 1051 rp = RetryPolicies.exponentialBackoffRetry(Long.SIZE - 2, 1052 kerberosMinSecondsBeforeRelogin, TimeUnit.MILLISECONDS); 1053 } 1054 try { 1055 nextRefresh = getNextTgtRenewalTime(tgtEndTime, now, rp); 1056 } catch (Exception e) { 1057 LOG.error("Exception when calculating next tgt renewal time", e); 1058 return; 1059 } 1060 metrics.renewalFailures.incr(); 1061 // retry until close enough to tgt endTime. 1062 if (now > nextRefresh) { 1063 LOG.error("TGT is expired. Aborting renew thread for {}.", 1064 getUserName()); 1065 return; 1066 } 1067 } 1068 } 1069 } 1070 }); 1071 t.setDaemon(true); 1072 t.setName("TGT Renewer for " + getUserName()); 1073 t.start(); 1074 } 1075 1076 /** 1077 * Get time for next login retry. This will allow the thread to retry with 1078 * exponential back-off, until tgt endtime. 1079 * Last retry is {@link #kerberosMinSecondsBeforeRelogin} before endtime. 1080 * 1081 * @param tgtEndTime EndTime of the tgt. 1082 * @param now Current time. 1083 * @param rp The retry policy. 1084 * @return Time for next login retry. 1085 */ 1086 @VisibleForTesting 1087 static long getNextTgtRenewalTime(final long tgtEndTime, final long now, 1088 final RetryPolicy rp) throws Exception { 1089 final long lastRetryTime = tgtEndTime - kerberosMinSecondsBeforeRelogin; 1090 final RetryPolicy.RetryAction ra = rp.shouldRetry(null, 1091 metrics.renewalFailures.value(), 0, false); 1092 return Math.min(lastRetryTime, now + ra.delayMillis); 1093 } 1094 1095 /** 1096 * Log a user in from a keytab file. Loads a user identity from a keytab 1097 * file and logs them in. They become the currently logged-in user. 1098 * @param user the principal name to load from the keytab 1099 * @param path the path to the keytab file 1100 * @throws IOException 1101 * @throws KerberosAuthException if it's a kerberos login exception. 1102 */ 1103 @InterfaceAudience.Public 1104 @InterfaceStability.Evolving 1105 public synchronized 1106 static void loginUserFromKeytab(String user, 1107 String path 1108 ) throws IOException { 1109 if (!isSecurityEnabled()) 1110 return; 1111 1112 keytabFile = path; 1113 keytabPrincipal = user; 1114 Subject subject = new Subject(); 1115 LoginContext login; 1116 long start = 0; 1117 try { 1118 login = newLoginContext(HadoopConfiguration.KEYTAB_KERBEROS_CONFIG_NAME, 1119 subject, new HadoopConfiguration()); 1120 start = Time.now(); 1121 login.login(); 1122 metrics.loginSuccess.add(Time.now() - start); 1123 loginUser = new UserGroupInformation(subject, false); 1124 loginUser.setLogin(login); 1125 loginUser.setAuthenticationMethod(AuthenticationMethod.KERBEROS); 1126 } catch (LoginException le) { 1127 if (start > 0) { 1128 metrics.loginFailure.add(Time.now() - start); 1129 } 1130 KerberosAuthException kae = new KerberosAuthException(LOGIN_FAILURE, le); 1131 kae.setUser(user); 1132 kae.setKeytabFile(path); 1133 throw kae; 1134 } 1135 LOG.info("Login successful for user " + keytabPrincipal 1136 + " using keytab file " + keytabFile); 1137 } 1138 1139 /** 1140 * Log the current user out who previously logged in using keytab. 1141 * This method assumes that the user logged in by calling 1142 * {@link #loginUserFromKeytab(String, String)}. 1143 * 1144 * @throws IOException 1145 * @throws KerberosAuthException if a failure occurred in logout, 1146 * or if the user did not log in by invoking loginUserFromKeyTab() before. 1147 */ 1148 @InterfaceAudience.Public 1149 @InterfaceStability.Evolving 1150 public void logoutUserFromKeytab() throws IOException { 1151 if (!isSecurityEnabled() || 1152 user.getAuthenticationMethod() != AuthenticationMethod.KERBEROS) { 1153 return; 1154 } 1155 LoginContext login = getLogin(); 1156 if (login == null || keytabFile == null) { 1157 throw new KerberosAuthException(MUST_FIRST_LOGIN_FROM_KEYTAB); 1158 } 1159 1160 try { 1161 if (LOG.isDebugEnabled()) { 1162 LOG.debug("Initiating logout for " + getUserName()); 1163 } 1164 synchronized (UserGroupInformation.class) { 1165 login.logout(); 1166 } 1167 } catch (LoginException le) { 1168 KerberosAuthException kae = new KerberosAuthException(LOGOUT_FAILURE, le); 1169 kae.setUser(user.toString()); 1170 kae.setKeytabFile(keytabFile); 1171 throw kae; 1172 } 1173 1174 LOG.info("Logout successful for user " + keytabPrincipal 1175 + " using keytab file " + keytabFile); 1176 } 1177 1178 /** 1179 * Re-login a user from keytab if TGT is expired or is close to expiry. 1180 * 1181 * @throws IOException 1182 * @throws KerberosAuthException if it's a kerberos login exception. 1183 */ 1184 public synchronized void checkTGTAndReloginFromKeytab() throws IOException { 1185 if (!isSecurityEnabled() 1186 || user.getAuthenticationMethod() != AuthenticationMethod.KERBEROS 1187 || !isKeytab) { 1188 return; 1189 } 1190 KerberosTicket tgt = getTGT(); 1191 if (tgt != null && !shouldRenewImmediatelyForTests && 1192 Time.now() < getRefreshTime(tgt)) { 1193 return; 1194 } 1195 reloginFromKeytab(); 1196 } 1197 1198 // if the first kerberos ticket is not TGT, then remove and destroy it since 1199 // the kerberos library of jdk always use the first kerberos ticket as TGT. 1200 // See HADOOP-13433 for more details. 1201 private void fixKerberosTicketOrder() { 1202 Set<Object> creds = getSubject().getPrivateCredentials(); 1203 synchronized (creds) { 1204 for (Iterator<Object> iter = creds.iterator(); iter.hasNext();) { 1205 Object cred = iter.next(); 1206 if (cred instanceof KerberosTicket) { 1207 KerberosTicket ticket = (KerberosTicket) cred; 1208 if (!ticket.getServer().getName().startsWith("krbtgt")) { 1209 LOG.warn( 1210 "The first kerberos ticket is not TGT" 1211 + "(the server principal is {}), remove and destroy it.", 1212 ticket.getServer()); 1213 iter.remove(); 1214 try { 1215 ticket.destroy(); 1216 } catch (DestroyFailedException e) { 1217 LOG.warn("destroy ticket failed", e); 1218 } 1219 } else { 1220 return; 1221 } 1222 } 1223 } 1224 } 1225 LOG.warn("Warning, no kerberos ticket found while attempting to renew ticket"); 1226 } 1227 1228 /** 1229 * Re-Login a user in from a keytab file. Loads a user identity from a keytab 1230 * file and logs them in. They become the currently logged-in user. This 1231 * method assumes that {@link #loginUserFromKeytab(String, String)} had 1232 * happened already. 1233 * The Subject field of this UserGroupInformation object is updated to have 1234 * the new credentials. 1235 * @throws IOException 1236 * @throws KerberosAuthException on a failure 1237 */ 1238 @InterfaceAudience.Public 1239 @InterfaceStability.Evolving 1240 public synchronized void reloginFromKeytab() throws IOException { 1241 if (!shouldRelogin() || !isKeytab) { 1242 return; 1243 } 1244 1245 long now = Time.now(); 1246 if (!shouldRenewImmediatelyForTests && !hasSufficientTimeElapsed(now)) { 1247 return; 1248 } 1249 1250 KerberosTicket tgt = getTGT(); 1251 //Return if TGT is valid and is not going to expire soon. 1252 if (tgt != null && !shouldRenewImmediatelyForTests && 1253 now < getRefreshTime(tgt)) { 1254 return; 1255 } 1256 1257 LoginContext login = getLogin(); 1258 if (login == null || keytabFile == null) { 1259 throw new KerberosAuthException(MUST_FIRST_LOGIN_FROM_KEYTAB); 1260 } 1261 1262 long start = 0; 1263 // register most recent relogin attempt 1264 user.setLastLogin(now); 1265 try { 1266 if (LOG.isDebugEnabled()) { 1267 LOG.debug("Initiating logout for " + getUserName()); 1268 } 1269 synchronized (UserGroupInformation.class) { 1270 // clear up the kerberos state. But the tokens are not cleared! As per 1271 // the Java kerberos login module code, only the kerberos credentials 1272 // are cleared 1273 login.logout(); 1274 // login and also update the subject field of this instance to 1275 // have the new credentials (pass it to the LoginContext constructor) 1276 login = newLoginContext( 1277 HadoopConfiguration.KEYTAB_KERBEROS_CONFIG_NAME, getSubject(), 1278 new HadoopConfiguration()); 1279 if (LOG.isDebugEnabled()) { 1280 LOG.debug("Initiating re-login for " + keytabPrincipal); 1281 } 1282 start = Time.now(); 1283 login.login(); 1284 fixKerberosTicketOrder(); 1285 metrics.loginSuccess.add(Time.now() - start); 1286 setLogin(login); 1287 } 1288 } catch (LoginException le) { 1289 if (start > 0) { 1290 metrics.loginFailure.add(Time.now() - start); 1291 } 1292 KerberosAuthException kae = new KerberosAuthException(LOGIN_FAILURE, le); 1293 kae.setPrincipal(keytabPrincipal); 1294 kae.setKeytabFile(keytabFile); 1295 throw kae; 1296 } 1297 } 1298 1299 /** 1300 * Re-Login a user in from the ticket cache. This 1301 * method assumes that login had happened already. 1302 * The Subject field of this UserGroupInformation object is updated to have 1303 * the new credentials. 1304 * @throws IOException 1305 * @throws KerberosAuthException on a failure 1306 */ 1307 @InterfaceAudience.Public 1308 @InterfaceStability.Evolving 1309 public synchronized void reloginFromTicketCache() throws IOException { 1310 if (!shouldRelogin() || !isKrbTkt) { 1311 return; 1312 } 1313 LoginContext login = getLogin(); 1314 if (login == null) { 1315 throw new KerberosAuthException(MUST_FIRST_LOGIN); 1316 } 1317 long now = Time.now(); 1318 if (!hasSufficientTimeElapsed(now)) { 1319 return; 1320 } 1321 // register most recent relogin attempt 1322 user.setLastLogin(now); 1323 try { 1324 if (LOG.isDebugEnabled()) { 1325 LOG.debug("Initiating logout for " + getUserName()); 1326 } 1327 //clear up the kerberos state. But the tokens are not cleared! As per 1328 //the Java kerberos login module code, only the kerberos credentials 1329 //are cleared 1330 login.logout(); 1331 //login and also update the subject field of this instance to 1332 //have the new credentials (pass it to the LoginContext constructor) 1333 login = 1334 newLoginContext(HadoopConfiguration.USER_KERBEROS_CONFIG_NAME, 1335 getSubject(), new HadoopConfiguration()); 1336 if (LOG.isDebugEnabled()) { 1337 LOG.debug("Initiating re-login for " + getUserName()); 1338 } 1339 login.login(); 1340 fixKerberosTicketOrder(); 1341 setLogin(login); 1342 } catch (LoginException le) { 1343 KerberosAuthException kae = new KerberosAuthException(LOGIN_FAILURE, le); 1344 kae.setUser(getUserName()); 1345 throw kae; 1346 } 1347 } 1348 1349 /** 1350 * Log a user in from a keytab file. Loads a user identity from a keytab 1351 * file and login them in. This new user does not affect the currently 1352 * logged-in user. 1353 * @param user the principal name to load from the keytab 1354 * @param path the path to the keytab file 1355 * @throws IOException if the keytab file can't be read 1356 */ 1357 public synchronized 1358 static UserGroupInformation loginUserFromKeytabAndReturnUGI(String user, 1359 String path 1360 ) throws IOException { 1361 if (!isSecurityEnabled()) 1362 return UserGroupInformation.getCurrentUser(); 1363 String oldKeytabFile = null; 1364 String oldKeytabPrincipal = null; 1365 1366 long start = 0; 1367 try { 1368 oldKeytabFile = keytabFile; 1369 oldKeytabPrincipal = keytabPrincipal; 1370 keytabFile = path; 1371 keytabPrincipal = user; 1372 Subject subject = new Subject(); 1373 1374 LoginContext login = newLoginContext( 1375 HadoopConfiguration.KEYTAB_KERBEROS_CONFIG_NAME, subject, 1376 new HadoopConfiguration()); 1377 1378 start = Time.now(); 1379 login.login(); 1380 metrics.loginSuccess.add(Time.now() - start); 1381 UserGroupInformation newLoginUser = 1382 new UserGroupInformation(subject, false); 1383 newLoginUser.setLogin(login); 1384 newLoginUser.setAuthenticationMethod(AuthenticationMethod.KERBEROS); 1385 1386 return newLoginUser; 1387 } catch (LoginException le) { 1388 if (start > 0) { 1389 metrics.loginFailure.add(Time.now() - start); 1390 } 1391 KerberosAuthException kae = new KerberosAuthException(LOGIN_FAILURE, le); 1392 kae.setUser(user); 1393 kae.setKeytabFile(path); 1394 throw kae; 1395 } finally { 1396 if(oldKeytabFile != null) keytabFile = oldKeytabFile; 1397 if(oldKeytabPrincipal != null) keytabPrincipal = oldKeytabPrincipal; 1398 } 1399 } 1400 1401 private boolean hasSufficientTimeElapsed(long now) { 1402 if (now - user.getLastLogin() < kerberosMinSecondsBeforeRelogin ) { 1403 LOG.warn("Not attempting to re-login since the last re-login was " + 1404 "attempted less than " + (kerberosMinSecondsBeforeRelogin/1000) + 1405 " seconds before. Last Login=" + user.getLastLogin()); 1406 return false; 1407 } 1408 return true; 1409 } 1410 1411 /** 1412 * Did the login happen via keytab 1413 * @return true or false 1414 */ 1415 @InterfaceAudience.Public 1416 @InterfaceStability.Evolving 1417 public synchronized static boolean isLoginKeytabBased() throws IOException { 1418 return getLoginUser().isKeytab; 1419 } 1420 1421 /** 1422 * Did the login happen via ticket cache 1423 * @return true or false 1424 */ 1425 public static boolean isLoginTicketBased() throws IOException { 1426 return getLoginUser().isKrbTkt; 1427 } 1428 1429 /** 1430 * Create a user from a login name. It is intended to be used for remote 1431 * users in RPC, since it won't have any credentials. 1432 * @param user the full user principal name, must not be empty or null 1433 * @return the UserGroupInformation for the remote user. 1434 */ 1435 @InterfaceAudience.Public 1436 @InterfaceStability.Evolving 1437 public static UserGroupInformation createRemoteUser(String user) { 1438 return createRemoteUser(user, AuthMethod.SIMPLE); 1439 } 1440 1441 /** 1442 * Create a user from a login name. It is intended to be used for remote 1443 * users in RPC, since it won't have any credentials. 1444 * @param user the full user principal name, must not be empty or null 1445 * @return the UserGroupInformation for the remote user. 1446 */ 1447 @InterfaceAudience.Public 1448 @InterfaceStability.Evolving 1449 public static UserGroupInformation createRemoteUser(String user, AuthMethod authMethod) { 1450 if (user == null || user.isEmpty()) { 1451 throw new IllegalArgumentException("Null user"); 1452 } 1453 Subject subject = new Subject(); 1454 subject.getPrincipals().add(new User(user)); 1455 UserGroupInformation result = new UserGroupInformation(subject, false); 1456 result.setAuthenticationMethod(authMethod); 1457 return result; 1458 } 1459 1460 /** 1461 * existing types of authentications' methods 1462 */ 1463 @InterfaceAudience.Public 1464 @InterfaceStability.Evolving 1465 public static enum AuthenticationMethod { 1466 // currently we support only one auth per method, but eventually a 1467 // subtype is needed to differentiate, ex. if digest is token or ldap 1468 SIMPLE(AuthMethod.SIMPLE, 1469 HadoopConfiguration.SIMPLE_CONFIG_NAME), 1470 KERBEROS(AuthMethod.KERBEROS, 1471 HadoopConfiguration.USER_KERBEROS_CONFIG_NAME), 1472 TOKEN(AuthMethod.TOKEN), 1473 CERTIFICATE(null), 1474 KERBEROS_SSL(null), 1475 PROXY(null); 1476 1477 private final AuthMethod authMethod; 1478 private final String loginAppName; 1479 1480 private AuthenticationMethod(AuthMethod authMethod) { 1481 this(authMethod, null); 1482 } 1483 private AuthenticationMethod(AuthMethod authMethod, String loginAppName) { 1484 this.authMethod = authMethod; 1485 this.loginAppName = loginAppName; 1486 } 1487 1488 public AuthMethod getAuthMethod() { 1489 return authMethod; 1490 } 1491 1492 String getLoginAppName() { 1493 if (loginAppName == null) { 1494 throw new UnsupportedOperationException( 1495 this + " login authentication is not supported"); 1496 } 1497 return loginAppName; 1498 } 1499 1500 public static AuthenticationMethod valueOf(AuthMethod authMethod) { 1501 for (AuthenticationMethod value : values()) { 1502 if (value.getAuthMethod() == authMethod) { 1503 return value; 1504 } 1505 } 1506 throw new IllegalArgumentException( 1507 "no authentication method for " + authMethod); 1508 } 1509 }; 1510 1511 /** 1512 * Create a proxy user using username of the effective user and the ugi of the 1513 * real user. 1514 * @param user 1515 * @param realUser 1516 * @return proxyUser ugi 1517 */ 1518 @InterfaceAudience.Public 1519 @InterfaceStability.Evolving 1520 public static UserGroupInformation createProxyUser(String user, 1521 UserGroupInformation realUser) { 1522 if (user == null || user.isEmpty()) { 1523 throw new IllegalArgumentException("Null user"); 1524 } 1525 if (realUser == null) { 1526 throw new IllegalArgumentException("Null real user"); 1527 } 1528 Subject subject = new Subject(); 1529 Set<Principal> principals = subject.getPrincipals(); 1530 principals.add(new User(user)); 1531 principals.add(new RealUser(realUser)); 1532 UserGroupInformation result =new UserGroupInformation(subject, false); 1533 result.setAuthenticationMethod(AuthenticationMethod.PROXY); 1534 return result; 1535 } 1536 1537 /** 1538 * get RealUser (vs. EffectiveUser) 1539 * @return realUser running over proxy user 1540 */ 1541 @InterfaceAudience.Public 1542 @InterfaceStability.Evolving 1543 public UserGroupInformation getRealUser() { 1544 for (RealUser p: subject.getPrincipals(RealUser.class)) { 1545 return p.getRealUser(); 1546 } 1547 return null; 1548 } 1549 1550 1551 1552 /** 1553 * This class is used for storing the groups for testing. It stores a local 1554 * map that has the translation of usernames to groups. 1555 */ 1556 private static class TestingGroups extends Groups { 1557 private final Map<String, List<String>> userToGroupsMapping = 1558 new HashMap<String,List<String>>(); 1559 private Groups underlyingImplementation; 1560 1561 private TestingGroups(Groups underlyingImplementation) { 1562 super(new org.apache.hadoop.conf.Configuration()); 1563 this.underlyingImplementation = underlyingImplementation; 1564 } 1565 1566 @Override 1567 public List<String> getGroups(String user) throws IOException { 1568 List<String> result = userToGroupsMapping.get(user); 1569 1570 if (result == null) { 1571 result = underlyingImplementation.getGroups(user); 1572 } 1573 1574 return result; 1575 } 1576 1577 private void setUserGroups(String user, String[] groups) { 1578 userToGroupsMapping.put(user, Arrays.asList(groups)); 1579 } 1580 } 1581 1582 /** 1583 * Create a UGI for testing HDFS and MapReduce 1584 * @param user the full user principal name 1585 * @param userGroups the names of the groups that the user belongs to 1586 * @return a fake user for running unit tests 1587 */ 1588 @InterfaceAudience.Public 1589 @InterfaceStability.Evolving 1590 public static UserGroupInformation createUserForTesting(String user, 1591 String[] userGroups) { 1592 ensureInitialized(); 1593 UserGroupInformation ugi = createRemoteUser(user); 1594 // make sure that the testing object is setup 1595 if (!(groups instanceof TestingGroups)) { 1596 groups = new TestingGroups(groups); 1597 } 1598 // add the user groups 1599 ((TestingGroups) groups).setUserGroups(ugi.getShortUserName(), userGroups); 1600 return ugi; 1601 } 1602 1603 1604 /** 1605 * Create a proxy user UGI for testing HDFS and MapReduce 1606 * 1607 * @param user 1608 * the full user principal name for effective user 1609 * @param realUser 1610 * UGI of the real user 1611 * @param userGroups 1612 * the names of the groups that the user belongs to 1613 * @return a fake user for running unit tests 1614 */ 1615 public static UserGroupInformation createProxyUserForTesting(String user, 1616 UserGroupInformation realUser, String[] userGroups) { 1617 ensureInitialized(); 1618 UserGroupInformation ugi = createProxyUser(user, realUser); 1619 // make sure that the testing object is setup 1620 if (!(groups instanceof TestingGroups)) { 1621 groups = new TestingGroups(groups); 1622 } 1623 // add the user groups 1624 ((TestingGroups) groups).setUserGroups(ugi.getShortUserName(), userGroups); 1625 return ugi; 1626 } 1627 1628 /** 1629 * Get the user's login name. 1630 * @return the user's name up to the first '/' or '@'. 1631 */ 1632 public String getShortUserName() { 1633 for (User p: subject.getPrincipals(User.class)) { 1634 return p.getShortName(); 1635 } 1636 return null; 1637 } 1638 1639 public String getPrimaryGroupName() throws IOException { 1640 List<String> groups = getGroups(); 1641 if (groups.isEmpty()) { 1642 throw new IOException("There is no primary group for UGI " + this); 1643 } 1644 return groups.get(0); 1645 } 1646 1647 /** 1648 * Get the user's full principal name. 1649 * @return the user's full principal name. 1650 */ 1651 @InterfaceAudience.Public 1652 @InterfaceStability.Evolving 1653 public String getUserName() { 1654 return user.getName(); 1655 } 1656 1657 /** 1658 * Add a TokenIdentifier to this UGI. The TokenIdentifier has typically been 1659 * authenticated by the RPC layer as belonging to the user represented by this 1660 * UGI. 1661 * 1662 * @param tokenId 1663 * tokenIdentifier to be added 1664 * @return true on successful add of new tokenIdentifier 1665 */ 1666 public synchronized boolean addTokenIdentifier(TokenIdentifier tokenId) { 1667 return subject.getPublicCredentials().add(tokenId); 1668 } 1669 1670 /** 1671 * Get the set of TokenIdentifiers belonging to this UGI 1672 * 1673 * @return the set of TokenIdentifiers belonging to this UGI 1674 */ 1675 public synchronized Set<TokenIdentifier> getTokenIdentifiers() { 1676 return subject.getPublicCredentials(TokenIdentifier.class); 1677 } 1678 1679 /** 1680 * Add a token to this UGI 1681 * 1682 * @param token Token to be added 1683 * @return true on successful add of new token 1684 */ 1685 public boolean addToken(Token<? extends TokenIdentifier> token) { 1686 return (token != null) ? addToken(token.getService(), token) : false; 1687 } 1688 1689 /** 1690 * Add a named token to this UGI 1691 * 1692 * @param alias Name of the token 1693 * @param token Token to be added 1694 * @return true on successful add of new token 1695 */ 1696 public boolean addToken(Text alias, Token<? extends TokenIdentifier> token) { 1697 synchronized (subject) { 1698 getCredentialsInternal().addToken(alias, token); 1699 return true; 1700 } 1701 } 1702 1703 /** 1704 * Obtain the collection of tokens associated with this user. 1705 * 1706 * @return an unmodifiable collection of tokens associated with user 1707 */ 1708 public Collection<Token<? extends TokenIdentifier>> getTokens() { 1709 synchronized (subject) { 1710 return Collections.unmodifiableCollection( 1711 new ArrayList<Token<?>>(getCredentialsInternal().getAllTokens())); 1712 } 1713 } 1714 1715 /** 1716 * Obtain the tokens in credentials form associated with this user. 1717 * 1718 * @return Credentials of tokens associated with this user 1719 */ 1720 public Credentials getCredentials() { 1721 synchronized (subject) { 1722 Credentials creds = new Credentials(getCredentialsInternal()); 1723 Iterator<Token<?>> iter = creds.getAllTokens().iterator(); 1724 while (iter.hasNext()) { 1725 if (iter.next() instanceof Token.PrivateToken) { 1726 iter.remove(); 1727 } 1728 } 1729 return creds; 1730 } 1731 } 1732 1733 /** 1734 * Add the given Credentials to this user. 1735 * @param credentials of tokens and secrets 1736 */ 1737 public void addCredentials(Credentials credentials) { 1738 synchronized (subject) { 1739 getCredentialsInternal().addAll(credentials); 1740 } 1741 } 1742 1743 private synchronized Credentials getCredentialsInternal() { 1744 final Credentials credentials; 1745 final Set<Credentials> credentialsSet = 1746 subject.getPrivateCredentials(Credentials.class); 1747 if (!credentialsSet.isEmpty()){ 1748 credentials = credentialsSet.iterator().next(); 1749 } else { 1750 credentials = new Credentials(); 1751 subject.getPrivateCredentials().add(credentials); 1752 } 1753 return credentials; 1754 } 1755 1756 /** 1757 * Get the group names for this user. {@ #getGroups(String)} is less 1758 * expensive alternative when checking for a contained element. 1759 * @return the list of users with the primary group first. If the command 1760 * fails, it returns an empty list. 1761 */ 1762 public String[] getGroupNames() { 1763 List<String> groups = getGroups(); 1764 return groups.toArray(new String[groups.size()]); 1765 } 1766 1767 /** 1768 * Get the group names for this user. 1769 * @return the list of users with the primary group first. If the command 1770 * fails, it returns an empty list. 1771 */ 1772 public List<String> getGroups() { 1773 ensureInitialized(); 1774 try { 1775 return groups.getGroups(getShortUserName()); 1776 } catch (IOException ie) { 1777 if (LOG.isDebugEnabled()) { 1778 LOG.debug("Failed to get groups for user " + getShortUserName() 1779 + " by " + ie); 1780 LOG.trace("TRACE", ie); 1781 } 1782 return Collections.emptyList(); 1783 } 1784 } 1785 1786 /** 1787 * Return the username. 1788 */ 1789 @Override 1790 public String toString() { 1791 StringBuilder sb = new StringBuilder(getUserName()); 1792 sb.append(" (auth:"+getAuthenticationMethod()+")"); 1793 if (getRealUser() != null) { 1794 sb.append(" via ").append(getRealUser().toString()); 1795 } 1796 return sb.toString(); 1797 } 1798 1799 /** 1800 * Sets the authentication method in the subject 1801 * 1802 * @param authMethod 1803 */ 1804 public synchronized 1805 void setAuthenticationMethod(AuthenticationMethod authMethod) { 1806 user.setAuthenticationMethod(authMethod); 1807 } 1808 1809 /** 1810 * Sets the authentication method in the subject 1811 * 1812 * @param authMethod 1813 */ 1814 public void setAuthenticationMethod(AuthMethod authMethod) { 1815 user.setAuthenticationMethod(AuthenticationMethod.valueOf(authMethod)); 1816 } 1817 1818 /** 1819 * Get the authentication method from the subject 1820 * 1821 * @return AuthenticationMethod in the subject, null if not present. 1822 */ 1823 public synchronized AuthenticationMethod getAuthenticationMethod() { 1824 return user.getAuthenticationMethod(); 1825 } 1826 1827 /** 1828 * Get the authentication method from the real user's subject. If there 1829 * is no real user, return the given user's authentication method. 1830 * 1831 * @return AuthenticationMethod in the subject, null if not present. 1832 */ 1833 public synchronized AuthenticationMethod getRealAuthenticationMethod() { 1834 UserGroupInformation ugi = getRealUser(); 1835 if (ugi == null) { 1836 ugi = this; 1837 } 1838 return ugi.getAuthenticationMethod(); 1839 } 1840 1841 /** 1842 * Returns the authentication method of a ugi. If the authentication method is 1843 * PROXY, returns the authentication method of the real user. 1844 * 1845 * @param ugi 1846 * @return AuthenticationMethod 1847 */ 1848 public static AuthenticationMethod getRealAuthenticationMethod( 1849 UserGroupInformation ugi) { 1850 AuthenticationMethod authMethod = ugi.getAuthenticationMethod(); 1851 if (authMethod == AuthenticationMethod.PROXY) { 1852 authMethod = ugi.getRealUser().getAuthenticationMethod(); 1853 } 1854 return authMethod; 1855 } 1856 1857 /** 1858 * Compare the subjects to see if they are equal to each other. 1859 */ 1860 @Override 1861 public boolean equals(Object o) { 1862 if (o == this) { 1863 return true; 1864 } else if (o == null || getClass() != o.getClass()) { 1865 return false; 1866 } else { 1867 return subject == ((UserGroupInformation) o).subject; 1868 } 1869 } 1870 1871 /** 1872 * Return the hash of the subject. 1873 */ 1874 @Override 1875 public int hashCode() { 1876 return System.identityHashCode(subject); 1877 } 1878 1879 /** 1880 * Get the underlying subject from this ugi. 1881 * @return the subject that represents this user. 1882 */ 1883 protected Subject getSubject() { 1884 return subject; 1885 } 1886 1887 /** 1888 * Run the given action as the user. 1889 * @param <T> the return type of the run method 1890 * @param action the method to execute 1891 * @return the value from the run method 1892 */ 1893 @InterfaceAudience.Public 1894 @InterfaceStability.Evolving 1895 public <T> T doAs(PrivilegedAction<T> action) { 1896 logPrivilegedAction(subject, action); 1897 return Subject.doAs(subject, action); 1898 } 1899 1900 /** 1901 * Run the given action as the user, potentially throwing an exception. 1902 * @param <T> the return type of the run method 1903 * @param action the method to execute 1904 * @return the value from the run method 1905 * @throws IOException if the action throws an IOException 1906 * @throws Error if the action throws an Error 1907 * @throws RuntimeException if the action throws a RuntimeException 1908 * @throws InterruptedException if the action throws an InterruptedException 1909 * @throws UndeclaredThrowableException if the action throws something else 1910 */ 1911 @InterfaceAudience.Public 1912 @InterfaceStability.Evolving 1913 public <T> T doAs(PrivilegedExceptionAction<T> action 1914 ) throws IOException, InterruptedException { 1915 try { 1916 logPrivilegedAction(subject, action); 1917 return Subject.doAs(subject, action); 1918 } catch (PrivilegedActionException pae) { 1919 Throwable cause = pae.getCause(); 1920 LOG.warn("PriviledgedActionException as:"+this+" cause:"+cause); 1921 if (cause instanceof IOException) { 1922 throw (IOException) cause; 1923 } else if (cause instanceof Error) { 1924 throw (Error) cause; 1925 } else if (cause instanceof RuntimeException) { 1926 throw (RuntimeException) cause; 1927 } else if (cause instanceof InterruptedException) { 1928 throw (InterruptedException) cause; 1929 } else { 1930 throw new UndeclaredThrowableException(cause); 1931 } 1932 } 1933 } 1934 1935 private void logPrivilegedAction(Subject subject, Object action) { 1936 if (LOG.isDebugEnabled()) { 1937 // would be nice if action included a descriptive toString() 1938 String where = new Throwable().getStackTrace()[2].toString(); 1939 LOG.debug("PrivilegedAction as:"+this+" from:"+where); 1940 } 1941 } 1942 1943 public static void logAllUserInfo(UserGroupInformation ugi) throws 1944 IOException { 1945 if (LOG.isDebugEnabled()) { 1946 LOG.debug("UGI: " + ugi); 1947 if (ugi.getRealUser() != null) { 1948 LOG.debug("+RealUGI: " + ugi.getRealUser()); 1949 } 1950 LOG.debug("+LoginUGI: " + ugi.getLoginUser()); 1951 for (Token<?> token : ugi.getTokens()) { 1952 LOG.debug("+UGI token:" + token); 1953 } 1954 } 1955 } 1956 1957 private void print() throws IOException { 1958 System.out.println("User: " + getUserName()); 1959 System.out.print("Group Ids: "); 1960 System.out.println(); 1961 String[] groups = getGroupNames(); 1962 System.out.print("Groups: "); 1963 for(int i=0; i < groups.length; i++) { 1964 System.out.print(groups[i] + " "); 1965 } 1966 System.out.println(); 1967 } 1968 1969 /** 1970 * A test method to print out the current user's UGI. 1971 * @param args if there are two arguments, read the user from the keytab 1972 * and print it out. 1973 * @throws Exception 1974 */ 1975 public static void main(String [] args) throws Exception { 1976 System.out.println("Getting UGI for current user"); 1977 UserGroupInformation ugi = getCurrentUser(); 1978 ugi.print(); 1979 System.out.println("UGI: " + ugi); 1980 System.out.println("Auth method " + ugi.user.getAuthenticationMethod()); 1981 System.out.println("Keytab " + ugi.isKeytab); 1982 System.out.println("============================================================"); 1983 1984 if (args.length == 2) { 1985 System.out.println("Getting UGI from keytab...."); 1986 loginUserFromKeytab(args[0], args[1]); 1987 getCurrentUser().print(); 1988 System.out.println("Keytab: " + ugi); 1989 System.out.println("Auth method " + loginUser.user.getAuthenticationMethod()); 1990 System.out.println("Keytab " + loginUser.isKeytab); 1991 } 1992 } 1993 1994}