001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 019package org.apache.hadoop.conf; 020 021import com.google.common.annotations.VisibleForTesting; 022 023import java.io.BufferedInputStream; 024import java.io.DataInput; 025import java.io.DataOutput; 026import java.io.File; 027import java.io.FileInputStream; 028import java.io.IOException; 029import java.io.InputStream; 030import java.io.InputStreamReader; 031import java.io.OutputStream; 032import java.io.OutputStreamWriter; 033import java.io.Reader; 034import java.io.Writer; 035import java.lang.ref.WeakReference; 036import java.net.InetSocketAddress; 037import java.net.JarURLConnection; 038import java.net.URL; 039import java.net.URLConnection; 040import java.util.ArrayList; 041import java.util.Arrays; 042import java.util.Collection; 043import java.util.Collections; 044import java.util.Enumeration; 045import java.util.HashMap; 046import java.util.HashSet; 047import java.util.Iterator; 048import java.util.LinkedList; 049import java.util.List; 050import java.util.ListIterator; 051import java.util.Map; 052import java.util.Map.Entry; 053import java.util.Properties; 054import java.util.Set; 055import java.util.StringTokenizer; 056import java.util.WeakHashMap; 057import java.util.concurrent.ConcurrentHashMap; 058import java.util.concurrent.CopyOnWriteArrayList; 059import java.util.regex.Matcher; 060import java.util.regex.Pattern; 061import java.util.regex.PatternSyntaxException; 062import java.util.concurrent.TimeUnit; 063import java.util.concurrent.atomic.AtomicBoolean; 064import java.util.concurrent.atomic.AtomicReference; 065 066import javax.xml.parsers.DocumentBuilder; 067import javax.xml.parsers.DocumentBuilderFactory; 068import javax.xml.parsers.ParserConfigurationException; 069import javax.xml.transform.Transformer; 070import javax.xml.transform.TransformerException; 071import javax.xml.transform.TransformerFactory; 072import javax.xml.transform.dom.DOMSource; 073import javax.xml.transform.stream.StreamResult; 074 075import org.apache.commons.collections.map.UnmodifiableMap; 076import org.apache.commons.logging.Log; 077import org.apache.commons.logging.LogFactory; 078import org.apache.hadoop.classification.InterfaceAudience; 079import org.apache.hadoop.classification.InterfaceStability; 080import org.apache.hadoop.fs.CommonConfigurationKeysPublic; 081import org.apache.hadoop.fs.FileSystem; 082import org.apache.hadoop.fs.Path; 083import org.apache.hadoop.fs.CommonConfigurationKeys; 084import org.apache.hadoop.io.Writable; 085import org.apache.hadoop.io.WritableUtils; 086import org.apache.hadoop.net.NetUtils; 087import org.apache.hadoop.security.alias.CredentialProvider; 088import org.apache.hadoop.security.alias.CredentialProvider.CredentialEntry; 089import org.apache.hadoop.security.alias.CredentialProviderFactory; 090import org.apache.hadoop.util.ReflectionUtils; 091import org.apache.hadoop.util.StringInterner; 092import org.apache.hadoop.util.StringUtils; 093import org.codehaus.jackson.JsonFactory; 094import org.codehaus.jackson.JsonGenerator; 095import org.w3c.dom.DOMException; 096import org.w3c.dom.Document; 097import org.w3c.dom.Element; 098import org.w3c.dom.Node; 099import org.w3c.dom.NodeList; 100import org.w3c.dom.Text; 101import org.xml.sax.SAXException; 102 103import com.google.common.base.Preconditions; 104import com.google.common.base.Strings; 105 106/** 107 * Provides access to configuration parameters. 108 * 109 * <h4 id="Resources">Resources</h4> 110 * 111 * <p>Configurations are specified by resources. A resource contains a set of 112 * name/value pairs as XML data. Each resource is named by either a 113 * <code>String</code> or by a {@link Path}. If named by a <code>String</code>, 114 * then the classpath is examined for a file with that name. If named by a 115 * <code>Path</code>, then the local filesystem is examined directly, without 116 * referring to the classpath. 117 * 118 * <p>Unless explicitly turned off, Hadoop by default specifies two 119 * resources, loaded in-order from the classpath: <ol> 120 * <li><tt> 121 * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml"> 122 * core-default.xml</a></tt>: Read-only defaults for hadoop.</li> 123 * <li><tt>core-site.xml</tt>: Site-specific configuration for a given hadoop 124 * installation.</li> 125 * </ol> 126 * Applications may add additional resources, which are loaded 127 * subsequent to these resources in the order they are added. 128 * 129 * <h4 id="FinalParams">Final Parameters</h4> 130 * 131 * <p>Configuration parameters may be declared <i>final</i>. 132 * Once a resource declares a value final, no subsequently-loaded 133 * resource can alter that value. 134 * For example, one might define a final parameter with: 135 * <tt><pre> 136 * <property> 137 * <name>dfs.hosts.include</name> 138 * <value>/etc/hadoop/conf/hosts.include</value> 139 * <b><final>true</final></b> 140 * </property></pre></tt> 141 * 142 * Administrators typically define parameters as final in 143 * <tt>core-site.xml</tt> for values that user applications may not alter. 144 * 145 * <h4 id="VariableExpansion">Variable Expansion</h4> 146 * 147 * <p>Value strings are first processed for <i>variable expansion</i>. The 148 * available properties are:<ol> 149 * <li>Other properties defined in this Configuration; and, if a name is 150 * undefined here,</li> 151 * <li>Properties in {@link System#getProperties()}.</li> 152 * </ol> 153 * 154 * <p>For example, if a configuration resource contains the following property 155 * definitions: 156 * <tt><pre> 157 * <property> 158 * <name>basedir</name> 159 * <value>/user/${<i>user.name</i>}</value> 160 * </property> 161 * 162 * <property> 163 * <name>tempdir</name> 164 * <value>${<i>basedir</i>}/tmp</value> 165 * </property></pre></tt> 166 * 167 * When <tt>conf.get("tempdir")</tt> is called, then <tt>${<i>basedir</i>}</tt> 168 * will be resolved to another property in this Configuration, while 169 * <tt>${<i>user.name</i>}</tt> would then ordinarily be resolved to the value 170 * of the System property with that name. 171 * By default, warnings will be given to any deprecated configuration 172 * parameters and these are suppressible by configuring 173 * <tt>log4j.logger.org.apache.hadoop.conf.Configuration.deprecation</tt> in 174 * log4j.properties file. 175 */ 176@InterfaceAudience.Public 177@InterfaceStability.Stable 178public class Configuration implements Iterable<Map.Entry<String,String>>, 179 Writable { 180 private static final Log LOG = 181 LogFactory.getLog(Configuration.class); 182 183 private static final Log LOG_DEPRECATION = 184 LogFactory.getLog("org.apache.hadoop.conf.Configuration.deprecation"); 185 186 private boolean quietmode = true; 187 188 private static final String DEFAULT_STRING_CHECK = 189 "testingforemptydefaultvalue"; 190 191 private boolean allowNullValueProperties = false; 192 193 private static class Resource { 194 private final Object resource; 195 private final String name; 196 197 public Resource(Object resource) { 198 this(resource, resource.toString()); 199 } 200 201 public Resource(Object resource, String name) { 202 this.resource = resource; 203 this.name = name; 204 } 205 206 public String getName(){ 207 return name; 208 } 209 210 public Object getResource() { 211 return resource; 212 } 213 214 @Override 215 public String toString() { 216 return name; 217 } 218 } 219 220 /** 221 * List of configuration resources. 222 */ 223 private ArrayList<Resource> resources = new ArrayList<Resource>(); 224 225 /** 226 * The value reported as the setting resource when a key is set 227 * by code rather than a file resource by dumpConfiguration. 228 */ 229 static final String UNKNOWN_RESOURCE = "Unknown"; 230 231 232 /** 233 * List of configuration parameters marked <b>final</b>. 234 */ 235 private Set<String> finalParameters = Collections.newSetFromMap( 236 new ConcurrentHashMap<String, Boolean>()); 237 238 private boolean loadDefaults = true; 239 240 /** 241 * Configuration objects 242 */ 243 private static final WeakHashMap<Configuration,Object> REGISTRY = 244 new WeakHashMap<Configuration,Object>(); 245 246 /** 247 * List of default Resources. Resources are loaded in the order of the list 248 * entries 249 */ 250 private static final CopyOnWriteArrayList<String> defaultResources = 251 new CopyOnWriteArrayList<String>(); 252 253 private static final Map<ClassLoader, Map<String, WeakReference<Class<?>>>> 254 CACHE_CLASSES = new WeakHashMap<ClassLoader, Map<String, WeakReference<Class<?>>>>(); 255 256 /** 257 * Sentinel value to store negative cache results in {@link #CACHE_CLASSES}. 258 */ 259 private static final Class<?> NEGATIVE_CACHE_SENTINEL = 260 NegativeCacheSentinel.class; 261 262 /** 263 * Stores the mapping of key to the resource which modifies or loads 264 * the key most recently 265 */ 266 private Map<String, String[]> updatingResource; 267 268 /** 269 * Class to keep the information about the keys which replace the deprecated 270 * ones. 271 * 272 * This class stores the new keys which replace the deprecated keys and also 273 * gives a provision to have a custom message for each of the deprecated key 274 * that is being replaced. It also provides method to get the appropriate 275 * warning message which can be logged whenever the deprecated key is used. 276 */ 277 private static class DeprecatedKeyInfo { 278 private final String[] newKeys; 279 private final String customMessage; 280 private final AtomicBoolean accessed = new AtomicBoolean(false); 281 282 DeprecatedKeyInfo(String[] newKeys, String customMessage) { 283 this.newKeys = newKeys; 284 this.customMessage = customMessage; 285 } 286 287 /** 288 * Method to provide the warning message. It gives the custom message if 289 * non-null, and default message otherwise. 290 * @param key the associated deprecated key. 291 * @return message that is to be logged when a deprecated key is used. 292 */ 293 private final String getWarningMessage(String key) { 294 String warningMessage; 295 if(customMessage == null) { 296 StringBuilder message = new StringBuilder(key); 297 String deprecatedKeySuffix = " is deprecated. Instead, use "; 298 message.append(deprecatedKeySuffix); 299 for (int i = 0; i < newKeys.length; i++) { 300 message.append(newKeys[i]); 301 if(i != newKeys.length-1) { 302 message.append(", "); 303 } 304 } 305 warningMessage = message.toString(); 306 } 307 else { 308 warningMessage = customMessage; 309 } 310 return warningMessage; 311 } 312 313 boolean getAndSetAccessed() { 314 return accessed.getAndSet(true); 315 } 316 317 public void clearAccessed() { 318 accessed.set(false); 319 } 320 } 321 322 /** 323 * A pending addition to the global set of deprecated keys. 324 */ 325 public static class DeprecationDelta { 326 private final String key; 327 private final String[] newKeys; 328 private final String customMessage; 329 330 DeprecationDelta(String key, String[] newKeys, String customMessage) { 331 Preconditions.checkNotNull(key); 332 Preconditions.checkNotNull(newKeys); 333 Preconditions.checkArgument(newKeys.length > 0); 334 this.key = key; 335 this.newKeys = newKeys; 336 this.customMessage = customMessage; 337 } 338 339 public DeprecationDelta(String key, String newKey, String customMessage) { 340 this(key, new String[] { newKey }, customMessage); 341 } 342 343 public DeprecationDelta(String key, String newKey) { 344 this(key, new String[] { newKey }, null); 345 } 346 347 public String getKey() { 348 return key; 349 } 350 351 public String[] getNewKeys() { 352 return newKeys; 353 } 354 355 public String getCustomMessage() { 356 return customMessage; 357 } 358 } 359 360 /** 361 * The set of all keys which are deprecated. 362 * 363 * DeprecationContext objects are immutable. 364 */ 365 private static class DeprecationContext { 366 /** 367 * Stores the deprecated keys, the new keys which replace the deprecated keys 368 * and custom message(if any provided). 369 */ 370 private final Map<String, DeprecatedKeyInfo> deprecatedKeyMap; 371 372 /** 373 * Stores a mapping from superseding keys to the keys which they deprecate. 374 */ 375 private final Map<String, String> reverseDeprecatedKeyMap; 376 377 /** 378 * Create a new DeprecationContext by copying a previous DeprecationContext 379 * and adding some deltas. 380 * 381 * @param other The previous deprecation context to copy, or null to start 382 * from nothing. 383 * @param deltas The deltas to apply. 384 */ 385 @SuppressWarnings("unchecked") 386 DeprecationContext(DeprecationContext other, DeprecationDelta[] deltas) { 387 HashMap<String, DeprecatedKeyInfo> newDeprecatedKeyMap = 388 new HashMap<String, DeprecatedKeyInfo>(); 389 HashMap<String, String> newReverseDeprecatedKeyMap = 390 new HashMap<String, String>(); 391 if (other != null) { 392 for (Entry<String, DeprecatedKeyInfo> entry : 393 other.deprecatedKeyMap.entrySet()) { 394 newDeprecatedKeyMap.put(entry.getKey(), entry.getValue()); 395 } 396 for (Entry<String, String> entry : 397 other.reverseDeprecatedKeyMap.entrySet()) { 398 newReverseDeprecatedKeyMap.put(entry.getKey(), entry.getValue()); 399 } 400 } 401 for (DeprecationDelta delta : deltas) { 402 if (!newDeprecatedKeyMap.containsKey(delta.getKey())) { 403 DeprecatedKeyInfo newKeyInfo = 404 new DeprecatedKeyInfo(delta.getNewKeys(), delta.getCustomMessage()); 405 newDeprecatedKeyMap.put(delta.key, newKeyInfo); 406 for (String newKey : delta.getNewKeys()) { 407 newReverseDeprecatedKeyMap.put(newKey, delta.key); 408 } 409 } 410 } 411 this.deprecatedKeyMap = 412 UnmodifiableMap.decorate(newDeprecatedKeyMap); 413 this.reverseDeprecatedKeyMap = 414 UnmodifiableMap.decorate(newReverseDeprecatedKeyMap); 415 } 416 417 Map<String, DeprecatedKeyInfo> getDeprecatedKeyMap() { 418 return deprecatedKeyMap; 419 } 420 421 Map<String, String> getReverseDeprecatedKeyMap() { 422 return reverseDeprecatedKeyMap; 423 } 424 } 425 426 private static DeprecationDelta[] defaultDeprecations = 427 new DeprecationDelta[] { 428 new DeprecationDelta("topology.script.file.name", 429 CommonConfigurationKeys.NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY), 430 new DeprecationDelta("topology.script.number.args", 431 CommonConfigurationKeys.NET_TOPOLOGY_SCRIPT_NUMBER_ARGS_KEY), 432 new DeprecationDelta("hadoop.configured.node.mapping", 433 CommonConfigurationKeys.NET_TOPOLOGY_CONFIGURED_NODE_MAPPING_KEY), 434 new DeprecationDelta("topology.node.switch.mapping.impl", 435 CommonConfigurationKeys.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY), 436 new DeprecationDelta("dfs.df.interval", 437 CommonConfigurationKeys.FS_DF_INTERVAL_KEY), 438 new DeprecationDelta("hadoop.native.lib", 439 CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_KEY), 440 new DeprecationDelta("fs.default.name", 441 CommonConfigurationKeys.FS_DEFAULT_NAME_KEY), 442 new DeprecationDelta("dfs.umaskmode", 443 CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY), 444 new DeprecationDelta("dfs.nfs.exports.allowed.hosts", 445 CommonConfigurationKeys.NFS_EXPORTS_ALLOWED_HOSTS_KEY) 446 }; 447 448 /** 449 * The global DeprecationContext. 450 */ 451 private static AtomicReference<DeprecationContext> deprecationContext = 452 new AtomicReference<DeprecationContext>( 453 new DeprecationContext(null, defaultDeprecations)); 454 455 /** 456 * Adds a set of deprecated keys to the global deprecations. 457 * 458 * This method is lockless. It works by means of creating a new 459 * DeprecationContext based on the old one, and then atomically swapping in 460 * the new context. If someone else updated the context in between us reading 461 * the old context and swapping in the new one, we try again until we win the 462 * race. 463 * 464 * @param deltas The deprecations to add. 465 */ 466 public static void addDeprecations(DeprecationDelta[] deltas) { 467 DeprecationContext prev, next; 468 do { 469 prev = deprecationContext.get(); 470 next = new DeprecationContext(prev, deltas); 471 } while (!deprecationContext.compareAndSet(prev, next)); 472 } 473 474 /** 475 * Adds the deprecated key to the global deprecation map. 476 * It does not override any existing entries in the deprecation map. 477 * This is to be used only by the developers in order to add deprecation of 478 * keys, and attempts to call this method after loading resources once, 479 * would lead to <tt>UnsupportedOperationException</tt> 480 * 481 * If a key is deprecated in favor of multiple keys, they are all treated as 482 * aliases of each other, and setting any one of them resets all the others 483 * to the new value. 484 * 485 * If you have multiple deprecation entries to add, it is more efficient to 486 * use #addDeprecations(DeprecationDelta[] deltas) instead. 487 * 488 * @param key 489 * @param newKeys 490 * @param customMessage 491 * @deprecated use {@link #addDeprecation(String key, String newKey, 492 String customMessage)} instead 493 */ 494 @Deprecated 495 public static void addDeprecation(String key, String[] newKeys, 496 String customMessage) { 497 addDeprecations(new DeprecationDelta[] { 498 new DeprecationDelta(key, newKeys, customMessage) 499 }); 500 } 501 502 /** 503 * Adds the deprecated key to the global deprecation map. 504 * It does not override any existing entries in the deprecation map. 505 * This is to be used only by the developers in order to add deprecation of 506 * keys, and attempts to call this method after loading resources once, 507 * would lead to <tt>UnsupportedOperationException</tt> 508 * 509 * If you have multiple deprecation entries to add, it is more efficient to 510 * use #addDeprecations(DeprecationDelta[] deltas) instead. 511 * 512 * @param key 513 * @param newKey 514 * @param customMessage 515 */ 516 public static void addDeprecation(String key, String newKey, 517 String customMessage) { 518 addDeprecation(key, new String[] {newKey}, customMessage); 519 } 520 521 /** 522 * Adds the deprecated key to the global deprecation map when no custom 523 * message is provided. 524 * It does not override any existing entries in the deprecation map. 525 * This is to be used only by the developers in order to add deprecation of 526 * keys, and attempts to call this method after loading resources once, 527 * would lead to <tt>UnsupportedOperationException</tt> 528 * 529 * If a key is deprecated in favor of multiple keys, they are all treated as 530 * aliases of each other, and setting any one of them resets all the others 531 * to the new value. 532 * 533 * If you have multiple deprecation entries to add, it is more efficient to 534 * use #addDeprecations(DeprecationDelta[] deltas) instead. 535 * 536 * @param key Key that is to be deprecated 537 * @param newKeys list of keys that take up the values of deprecated key 538 * @deprecated use {@link #addDeprecation(String key, String newKey)} instead 539 */ 540 @Deprecated 541 public static void addDeprecation(String key, String[] newKeys) { 542 addDeprecation(key, newKeys, null); 543 } 544 545 /** 546 * Adds the deprecated key to the global deprecation map when no custom 547 * message is provided. 548 * It does not override any existing entries in the deprecation map. 549 * This is to be used only by the developers in order to add deprecation of 550 * keys, and attempts to call this method after loading resources once, 551 * would lead to <tt>UnsupportedOperationException</tt> 552 * 553 * If you have multiple deprecation entries to add, it is more efficient to 554 * use #addDeprecations(DeprecationDelta[] deltas) instead. 555 * 556 * @param key Key that is to be deprecated 557 * @param newKey key that takes up the value of deprecated key 558 */ 559 public static void addDeprecation(String key, String newKey) { 560 addDeprecation(key, new String[] {newKey}, null); 561 } 562 563 /** 564 * checks whether the given <code>key</code> is deprecated. 565 * 566 * @param key the parameter which is to be checked for deprecation 567 * @return <code>true</code> if the key is deprecated and 568 * <code>false</code> otherwise. 569 */ 570 public static boolean isDeprecated(String key) { 571 return deprecationContext.get().getDeprecatedKeyMap().containsKey(key); 572 } 573 574 /** 575 * Sets all deprecated properties that are not currently set but have a 576 * corresponding new property that is set. Useful for iterating the 577 * properties when all deprecated properties for currently set properties 578 * need to be present. 579 */ 580 public void setDeprecatedProperties() { 581 DeprecationContext deprecations = deprecationContext.get(); 582 Properties props = getProps(); 583 Properties overlay = getOverlay(); 584 for (Map.Entry<String, DeprecatedKeyInfo> entry : 585 deprecations.getDeprecatedKeyMap().entrySet()) { 586 String depKey = entry.getKey(); 587 if (!overlay.contains(depKey)) { 588 for (String newKey : entry.getValue().newKeys) { 589 String val = overlay.getProperty(newKey); 590 if (val != null) { 591 props.setProperty(depKey, val); 592 overlay.setProperty(depKey, val); 593 break; 594 } 595 } 596 } 597 } 598 } 599 600 /** 601 * Checks for the presence of the property <code>name</code> in the 602 * deprecation map. Returns the first of the list of new keys if present 603 * in the deprecation map or the <code>name</code> itself. If the property 604 * is not presently set but the property map contains an entry for the 605 * deprecated key, the value of the deprecated key is set as the value for 606 * the provided property name. 607 * 608 * @param name the property name 609 * @return the first property in the list of properties mapping 610 * the <code>name</code> or the <code>name</code> itself. 611 */ 612 private String[] handleDeprecation(DeprecationContext deprecations, 613 String name) { 614 if (null != name) { 615 name = name.trim(); 616 } 617 ArrayList<String > names = new ArrayList<String>(); 618 if (isDeprecated(name)) { 619 DeprecatedKeyInfo keyInfo = deprecations.getDeprecatedKeyMap().get(name); 620 warnOnceIfDeprecated(deprecations, name); 621 for (String newKey : keyInfo.newKeys) { 622 if(newKey != null) { 623 names.add(newKey); 624 } 625 } 626 } 627 if(names.size() == 0) { 628 names.add(name); 629 } 630 for(String n : names) { 631 String deprecatedKey = deprecations.getReverseDeprecatedKeyMap().get(n); 632 if (deprecatedKey != null && !getOverlay().containsKey(n) && 633 getOverlay().containsKey(deprecatedKey)) { 634 getProps().setProperty(n, getOverlay().getProperty(deprecatedKey)); 635 getOverlay().setProperty(n, getOverlay().getProperty(deprecatedKey)); 636 } 637 } 638 return names.toArray(new String[names.size()]); 639 } 640 641 private void handleDeprecation() { 642 LOG.debug("Handling deprecation for all properties in config..."); 643 DeprecationContext deprecations = deprecationContext.get(); 644 Set<Object> keys = new HashSet<Object>(); 645 keys.addAll(getProps().keySet()); 646 for (Object item: keys) { 647 LOG.debug("Handling deprecation for " + (String)item); 648 handleDeprecation(deprecations, (String)item); 649 } 650 } 651 652 static{ 653 //print deprecation warning if hadoop-site.xml is found in classpath 654 ClassLoader cL = Thread.currentThread().getContextClassLoader(); 655 if (cL == null) { 656 cL = Configuration.class.getClassLoader(); 657 } 658 if(cL.getResource("hadoop-site.xml")!=null) { 659 LOG.warn("DEPRECATED: hadoop-site.xml found in the classpath. " + 660 "Usage of hadoop-site.xml is deprecated. Instead use core-site.xml, " 661 + "mapred-site.xml and hdfs-site.xml to override properties of " + 662 "core-default.xml, mapred-default.xml and hdfs-default.xml " + 663 "respectively"); 664 } 665 addDefaultResource("core-default.xml"); 666 addDefaultResource("core-site.xml"); 667 } 668 669 private Properties properties; 670 private Properties overlay; 671 private ClassLoader classLoader; 672 { 673 classLoader = Thread.currentThread().getContextClassLoader(); 674 if (classLoader == null) { 675 classLoader = Configuration.class.getClassLoader(); 676 } 677 } 678 679 /** A new configuration. */ 680 public Configuration() { 681 this(true); 682 } 683 684 /** A new configuration where the behavior of reading from the default 685 * resources can be turned off. 686 * 687 * If the parameter {@code loadDefaults} is false, the new instance 688 * will not load resources from the default files. 689 * @param loadDefaults specifies whether to load from the default files 690 */ 691 public Configuration(boolean loadDefaults) { 692 this.loadDefaults = loadDefaults; 693 updatingResource = new ConcurrentHashMap<String, String[]>(); 694 synchronized(Configuration.class) { 695 REGISTRY.put(this, null); 696 } 697 } 698 699 /** 700 * A new configuration with the same settings cloned from another. 701 * 702 * @param other the configuration from which to clone settings. 703 */ 704 @SuppressWarnings("unchecked") 705 public Configuration(Configuration other) { 706 this.resources = (ArrayList<Resource>) other.resources.clone(); 707 synchronized(other) { 708 if (other.properties != null) { 709 this.properties = (Properties)other.properties.clone(); 710 } 711 712 if (other.overlay!=null) { 713 this.overlay = (Properties)other.overlay.clone(); 714 } 715 716 this.updatingResource = new ConcurrentHashMap<String, String[]>( 717 other.updatingResource); 718 this.finalParameters = Collections.newSetFromMap( 719 new ConcurrentHashMap<String, Boolean>()); 720 this.finalParameters.addAll(other.finalParameters); 721 } 722 723 synchronized(Configuration.class) { 724 REGISTRY.put(this, null); 725 } 726 this.classLoader = other.classLoader; 727 this.loadDefaults = other.loadDefaults; 728 setQuietMode(other.getQuietMode()); 729 } 730 731 /** 732 * Add a default resource. Resources are loaded in the order of the resources 733 * added. 734 * @param name file name. File should be present in the classpath. 735 */ 736 public static synchronized void addDefaultResource(String name) { 737 if(!defaultResources.contains(name)) { 738 defaultResources.add(name); 739 for(Configuration conf : REGISTRY.keySet()) { 740 if(conf.loadDefaults) { 741 conf.reloadConfiguration(); 742 } 743 } 744 } 745 } 746 747 /** 748 * Add a configuration resource. 749 * 750 * The properties of this resource will override properties of previously 751 * added resources, unless they were marked <a href="#Final">final</a>. 752 * 753 * @param name resource to be added, the classpath is examined for a file 754 * with that name. 755 */ 756 public void addResource(String name) { 757 addResourceObject(new Resource(name)); 758 } 759 760 /** 761 * Add a configuration resource. 762 * 763 * The properties of this resource will override properties of previously 764 * added resources, unless they were marked <a href="#Final">final</a>. 765 * 766 * @param url url of the resource to be added, the local filesystem is 767 * examined directly to find the resource, without referring to 768 * the classpath. 769 */ 770 public void addResource(URL url) { 771 addResourceObject(new Resource(url)); 772 } 773 774 /** 775 * Add a configuration resource. 776 * 777 * The properties of this resource will override properties of previously 778 * added resources, unless they were marked <a href="#Final">final</a>. 779 * 780 * @param file file-path of resource to be added, the local filesystem is 781 * examined directly to find the resource, without referring to 782 * the classpath. 783 */ 784 public void addResource(Path file) { 785 addResourceObject(new Resource(file)); 786 } 787 788 /** 789 * Add a configuration resource. 790 * 791 * The properties of this resource will override properties of previously 792 * added resources, unless they were marked <a href="#Final">final</a>. 793 * 794 * WARNING: The contents of the InputStream will be cached, by this method. 795 * So use this sparingly because it does increase the memory consumption. 796 * 797 * @param in InputStream to deserialize the object from. In will be read from 798 * when a get or set is called next. After it is read the stream will be 799 * closed. 800 */ 801 public void addResource(InputStream in) { 802 addResourceObject(new Resource(in)); 803 } 804 805 /** 806 * Add a configuration resource. 807 * 808 * The properties of this resource will override properties of previously 809 * added resources, unless they were marked <a href="#Final">final</a>. 810 * 811 * @param in InputStream to deserialize the object from. 812 * @param name the name of the resource because InputStream.toString is not 813 * very descriptive some times. 814 */ 815 public void addResource(InputStream in, String name) { 816 addResourceObject(new Resource(in, name)); 817 } 818 819 /** 820 * Add a configuration resource. 821 * 822 * The properties of this resource will override properties of previously 823 * added resources, unless they were marked <a href="#Final">final</a>. 824 * 825 * @param conf Configuration object from which to load properties 826 */ 827 public void addResource(Configuration conf) { 828 addResourceObject(new Resource(conf.getProps())); 829 } 830 831 832 833 /** 834 * Reload configuration from previously added resources. 835 * 836 * This method will clear all the configuration read from the added 837 * resources, and final parameters. This will make the resources to 838 * be read again before accessing the values. Values that are added 839 * via set methods will overlay values read from the resources. 840 */ 841 public synchronized void reloadConfiguration() { 842 properties = null; // trigger reload 843 finalParameters.clear(); // clear site-limits 844 } 845 846 private synchronized void addResourceObject(Resource resource) { 847 resources.add(resource); // add to resources 848 reloadConfiguration(); 849 } 850 851 private static final int MAX_SUBST = 20; 852 853 private static final int SUB_START_IDX = 0; 854 private static final int SUB_END_IDX = SUB_START_IDX + 1; 855 856 /** 857 * This is a manual implementation of the following regex 858 * "\\$\\{[^\\}\\$\u0020]+\\}". It can be 15x more efficient than 859 * a regex matcher as demonstrated by HADOOP-11506. This is noticeable with 860 * Hadoop apps building on the assumption Configuration#get is an O(1) 861 * hash table lookup, especially when the eval is a long string. 862 * 863 * @param eval a string that may contain variables requiring expansion. 864 * @return a 2-element int array res such that 865 * eval.substring(res[0], res[1]) is "var" for the left-most occurrence of 866 * ${var} in eval. If no variable is found -1, -1 is returned. 867 */ 868 private static int[] findSubVariable(String eval) { 869 int[] result = {-1, -1}; 870 871 int matchStart; 872 int leftBrace; 873 874 // scanning for a brace first because it's less frequent than $ 875 // that can occur in nested class names 876 // 877 match_loop: 878 for (matchStart = 1, leftBrace = eval.indexOf('{', matchStart); 879 // minimum left brace position (follows '$') 880 leftBrace > 0 881 // right brace of a smallest valid expression "${c}" 882 && leftBrace + "{c".length() < eval.length(); 883 leftBrace = eval.indexOf('{', matchStart)) { 884 int matchedLen = 0; 885 if (eval.charAt(leftBrace - 1) == '$') { 886 int subStart = leftBrace + 1; // after '{' 887 for (int i = subStart; i < eval.length(); i++) { 888 switch (eval.charAt(i)) { 889 case '}': 890 if (matchedLen > 0) { // match 891 result[SUB_START_IDX] = subStart; 892 result[SUB_END_IDX] = subStart + matchedLen; 893 break match_loop; 894 } 895 // fall through to skip 1 char 896 case ' ': 897 case '$': 898 matchStart = i + 1; 899 continue match_loop; 900 default: 901 matchedLen++; 902 } 903 } 904 // scanned from "${" to the end of eval, and no reset via ' ', '$': 905 // no match! 906 break match_loop; 907 } else { 908 // not a start of a variable 909 // 910 matchStart = leftBrace + 1; 911 } 912 } 913 return result; 914 } 915 916 /** 917 * Attempts to repeatedly expand the value {@code expr} by replacing the 918 * left-most substring of the form "${var}" in the following precedence order 919 * <ol> 920 * <li>by the value of the Java system property "var" if defined</li> 921 * <li>by the value of the configuration key "var" if defined</li> 922 * </ol> 923 * 924 * If var is unbounded the current state of expansion "prefix${var}suffix" is 925 * returned. 926 * 927 * @param expr the literal value of a config key 928 * @return null if expr is null, otherwise the value resulting from expanding 929 * expr using the algorithm above. 930 * @throws IllegalArgumentException when more than 931 * {@link Configuration#MAX_SUBST} replacements are required 932 */ 933 private String substituteVars(String expr) { 934 if (expr == null) { 935 return null; 936 } 937 String eval = expr; 938 for (int s = 0; s < MAX_SUBST; s++) { 939 final int[] varBounds = findSubVariable(eval); 940 if (varBounds[SUB_START_IDX] == -1) { 941 return eval; 942 } 943 final String var = eval.substring(varBounds[SUB_START_IDX], 944 varBounds[SUB_END_IDX]); 945 String val = null; 946 try { 947 val = System.getProperty(var); 948 } catch(SecurityException se) { 949 LOG.warn("Unexpected SecurityException in Configuration", se); 950 } 951 if (val == null) { 952 val = getRaw(var); 953 } 954 if (val == null) { 955 return eval; // return literal ${var}: var is unbound 956 } 957 final int dollar = varBounds[SUB_START_IDX] - "${".length(); 958 final int afterRightBrace = varBounds[SUB_END_IDX] + "}".length(); 959 // substitute 960 eval = eval.substring(0, dollar) 961 + val 962 + eval.substring(afterRightBrace); 963 } 964 throw new IllegalStateException("Variable substitution depth too large: " 965 + MAX_SUBST + " " + expr); 966 } 967 968 /** 969 * Get the value of the <code>name</code> property, <code>null</code> if 970 * no such property exists. If the key is deprecated, it returns the value of 971 * the first key which replaces the deprecated key and is not null. 972 * 973 * Values are processed for <a href="#VariableExpansion">variable expansion</a> 974 * before being returned. 975 * 976 * @param name the property name, will be trimmed before get value. 977 * @return the value of the <code>name</code> or its replacing property, 978 * or null if no such property exists. 979 */ 980 public String get(String name) { 981 String[] names = handleDeprecation(deprecationContext.get(), name); 982 String result = null; 983 for(String n : names) { 984 result = substituteVars(getProps().getProperty(n)); 985 } 986 return result; 987 } 988 989 /** 990 * Set Configuration to allow keys without values during setup. Intended 991 * for use during testing. 992 * 993 * @param val If true, will allow Configuration to store keys without values 994 */ 995 @VisibleForTesting 996 public void setAllowNullValueProperties( boolean val ) { 997 this.allowNullValueProperties = val; 998 } 999 1000 /** 1001 * Return existence of the <code>name</code> property, but only for 1002 * names which have no valid value, usually non-existent or commented 1003 * out in XML. 1004 * 1005 * @param name the property name 1006 * @return true if the property <code>name</code> exists without value 1007 */ 1008 @VisibleForTesting 1009 public boolean onlyKeyExists(String name) { 1010 String[] names = handleDeprecation(deprecationContext.get(), name); 1011 for(String n : names) { 1012 if ( getProps().getProperty(n,DEFAULT_STRING_CHECK) 1013 .equals(DEFAULT_STRING_CHECK) ) { 1014 return true; 1015 } 1016 } 1017 return false; 1018 } 1019 1020 /** 1021 * Get the value of the <code>name</code> property as a trimmed <code>String</code>, 1022 * <code>null</code> if no such property exists. 1023 * If the key is deprecated, it returns the value of 1024 * the first key which replaces the deprecated key and is not null 1025 * 1026 * Values are processed for <a href="#VariableExpansion">variable expansion</a> 1027 * before being returned. 1028 * 1029 * @param name the property name. 1030 * @return the value of the <code>name</code> or its replacing property, 1031 * or null if no such property exists. 1032 */ 1033 public String getTrimmed(String name) { 1034 String value = get(name); 1035 1036 if (null == value) { 1037 return null; 1038 } else { 1039 return value.trim(); 1040 } 1041 } 1042 1043 /** 1044 * Get the value of the <code>name</code> property as a trimmed <code>String</code>, 1045 * <code>defaultValue</code> if no such property exists. 1046 * See @{Configuration#getTrimmed} for more details. 1047 * 1048 * @param name the property name. 1049 * @param defaultValue the property default value. 1050 * @return the value of the <code>name</code> or defaultValue 1051 * if it is not set. 1052 */ 1053 public String getTrimmed(String name, String defaultValue) { 1054 String ret = getTrimmed(name); 1055 return ret == null ? defaultValue : ret; 1056 } 1057 1058 /** 1059 * Get the value of the <code>name</code> property, without doing 1060 * <a href="#VariableExpansion">variable expansion</a>.If the key is 1061 * deprecated, it returns the value of the first key which replaces 1062 * the deprecated key and is not null. 1063 * 1064 * @param name the property name. 1065 * @return the value of the <code>name</code> property or 1066 * its replacing property and null if no such property exists. 1067 */ 1068 public String getRaw(String name) { 1069 String[] names = handleDeprecation(deprecationContext.get(), name); 1070 String result = null; 1071 for(String n : names) { 1072 result = getProps().getProperty(n); 1073 } 1074 return result; 1075 } 1076 1077 /** 1078 * Returns alternative names (non-deprecated keys or previously-set deprecated keys) 1079 * for a given non-deprecated key. 1080 * If the given key is deprecated, return null. 1081 * 1082 * @param name property name. 1083 * @return alternative names. 1084 */ 1085 private String[] getAlternativeNames(String name) { 1086 String altNames[] = null; 1087 DeprecatedKeyInfo keyInfo = null; 1088 DeprecationContext cur = deprecationContext.get(); 1089 String depKey = cur.getReverseDeprecatedKeyMap().get(name); 1090 if(depKey != null) { 1091 keyInfo = cur.getDeprecatedKeyMap().get(depKey); 1092 if(keyInfo.newKeys.length > 0) { 1093 if(getProps().containsKey(depKey)) { 1094 //if deprecated key is previously set explicitly 1095 List<String> list = new ArrayList<String>(); 1096 list.addAll(Arrays.asList(keyInfo.newKeys)); 1097 list.add(depKey); 1098 altNames = list.toArray(new String[list.size()]); 1099 } 1100 else { 1101 altNames = keyInfo.newKeys; 1102 } 1103 } 1104 } 1105 return altNames; 1106 } 1107 1108 /** 1109 * Set the <code>value</code> of the <code>name</code> property. If 1110 * <code>name</code> is deprecated or there is a deprecated name associated to it, 1111 * it sets the value to both names. Name will be trimmed before put into 1112 * configuration. 1113 * 1114 * @param name property name. 1115 * @param value property value. 1116 */ 1117 public void set(String name, String value) { 1118 set(name, value, null); 1119 } 1120 1121 /** 1122 * Set the <code>value</code> of the <code>name</code> property. If 1123 * <code>name</code> is deprecated, it also sets the <code>value</code> to 1124 * the keys that replace the deprecated key. Name will be trimmed before put 1125 * into configuration. 1126 * 1127 * @param name property name. 1128 * @param value property value. 1129 * @param source the place that this configuration value came from 1130 * (For debugging). 1131 * @throws IllegalArgumentException when the value or name is null. 1132 */ 1133 public void set(String name, String value, String source) { 1134 Preconditions.checkArgument( 1135 name != null, 1136 "Property name must not be null"); 1137 Preconditions.checkArgument( 1138 value != null, 1139 "The value of property " + name + " must not be null"); 1140 name = name.trim(); 1141 DeprecationContext deprecations = deprecationContext.get(); 1142 if (deprecations.getDeprecatedKeyMap().isEmpty()) { 1143 getProps(); 1144 } 1145 getOverlay().setProperty(name, value); 1146 getProps().setProperty(name, value); 1147 String newSource = (source == null ? "programatically" : source); 1148 1149 if (!isDeprecated(name)) { 1150 updatingResource.put(name, new String[] {newSource}); 1151 String[] altNames = getAlternativeNames(name); 1152 if(altNames != null) { 1153 for(String n: altNames) { 1154 if(!n.equals(name)) { 1155 getOverlay().setProperty(n, value); 1156 getProps().setProperty(n, value); 1157 updatingResource.put(n, new String[] {newSource}); 1158 } 1159 } 1160 } 1161 } 1162 else { 1163 String[] names = handleDeprecation(deprecationContext.get(), name); 1164 String altSource = "because " + name + " is deprecated"; 1165 for(String n : names) { 1166 getOverlay().setProperty(n, value); 1167 getProps().setProperty(n, value); 1168 updatingResource.put(n, new String[] {altSource}); 1169 } 1170 } 1171 } 1172 1173 private void warnOnceIfDeprecated(DeprecationContext deprecations, String name) { 1174 DeprecatedKeyInfo keyInfo = deprecations.getDeprecatedKeyMap().get(name); 1175 if (keyInfo != null && !keyInfo.getAndSetAccessed()) { 1176 LOG_DEPRECATION.info(keyInfo.getWarningMessage(name)); 1177 } 1178 } 1179 1180 /** 1181 * Unset a previously set property. 1182 */ 1183 public synchronized void unset(String name) { 1184 String[] names = null; 1185 if (!isDeprecated(name)) { 1186 names = getAlternativeNames(name); 1187 if(names == null) { 1188 names = new String[]{name}; 1189 } 1190 } 1191 else { 1192 names = handleDeprecation(deprecationContext.get(), name); 1193 } 1194 1195 for(String n: names) { 1196 getOverlay().remove(n); 1197 getProps().remove(n); 1198 } 1199 } 1200 1201 /** 1202 * Sets a property if it is currently unset. 1203 * @param name the property name 1204 * @param value the new value 1205 */ 1206 public synchronized void setIfUnset(String name, String value) { 1207 if (get(name) == null) { 1208 set(name, value); 1209 } 1210 } 1211 1212 private synchronized Properties getOverlay() { 1213 if (overlay==null){ 1214 overlay=new Properties(); 1215 } 1216 return overlay; 1217 } 1218 1219 /** 1220 * Get the value of the <code>name</code>. If the key is deprecated, 1221 * it returns the value of the first key which replaces the deprecated key 1222 * and is not null. 1223 * If no such property exists, 1224 * then <code>defaultValue</code> is returned. 1225 * 1226 * @param name property name, will be trimmed before get value. 1227 * @param defaultValue default value. 1228 * @return property value, or <code>defaultValue</code> if the property 1229 * doesn't exist. 1230 */ 1231 public String get(String name, String defaultValue) { 1232 String[] names = handleDeprecation(deprecationContext.get(), name); 1233 String result = null; 1234 for(String n : names) { 1235 result = substituteVars(getProps().getProperty(n, defaultValue)); 1236 } 1237 return result; 1238 } 1239 1240 /** 1241 * Get the value of the <code>name</code> property as an <code>int</code>. 1242 * 1243 * If no such property exists, the provided default value is returned, 1244 * or if the specified value is not a valid <code>int</code>, 1245 * then an error is thrown. 1246 * 1247 * @param name property name. 1248 * @param defaultValue default value. 1249 * @throws NumberFormatException when the value is invalid 1250 * @return property value as an <code>int</code>, 1251 * or <code>defaultValue</code>. 1252 */ 1253 public int getInt(String name, int defaultValue) { 1254 String valueString = getTrimmed(name); 1255 if (valueString == null) 1256 return defaultValue; 1257 String hexString = getHexDigits(valueString); 1258 if (hexString != null) { 1259 return Integer.parseInt(hexString, 16); 1260 } 1261 return Integer.parseInt(valueString); 1262 } 1263 1264 /** 1265 * Get the value of the <code>name</code> property as a set of comma-delimited 1266 * <code>int</code> values. 1267 * 1268 * If no such property exists, an empty array is returned. 1269 * 1270 * @param name property name 1271 * @return property value interpreted as an array of comma-delimited 1272 * <code>int</code> values 1273 */ 1274 public int[] getInts(String name) { 1275 String[] strings = getTrimmedStrings(name); 1276 int[] ints = new int[strings.length]; 1277 for (int i = 0; i < strings.length; i++) { 1278 ints[i] = Integer.parseInt(strings[i]); 1279 } 1280 return ints; 1281 } 1282 1283 /** 1284 * Set the value of the <code>name</code> property to an <code>int</code>. 1285 * 1286 * @param name property name. 1287 * @param value <code>int</code> value of the property. 1288 */ 1289 public void setInt(String name, int value) { 1290 set(name, Integer.toString(value)); 1291 } 1292 1293 1294 /** 1295 * Get the value of the <code>name</code> property as a <code>long</code>. 1296 * If no such property exists, the provided default value is returned, 1297 * or if the specified value is not a valid <code>long</code>, 1298 * then an error is thrown. 1299 * 1300 * @param name property name. 1301 * @param defaultValue default value. 1302 * @throws NumberFormatException when the value is invalid 1303 * @return property value as a <code>long</code>, 1304 * or <code>defaultValue</code>. 1305 */ 1306 public long getLong(String name, long defaultValue) { 1307 String valueString = getTrimmed(name); 1308 if (valueString == null) 1309 return defaultValue; 1310 String hexString = getHexDigits(valueString); 1311 if (hexString != null) { 1312 return Long.parseLong(hexString, 16); 1313 } 1314 return Long.parseLong(valueString); 1315 } 1316 1317 /** 1318 * Get the value of the <code>name</code> property as a <code>long</code> or 1319 * human readable format. If no such property exists, the provided default 1320 * value is returned, or if the specified value is not a valid 1321 * <code>long</code> or human readable format, then an error is thrown. You 1322 * can use the following suffix (case insensitive): k(kilo), m(mega), g(giga), 1323 * t(tera), p(peta), e(exa) 1324 * 1325 * @param name property name. 1326 * @param defaultValue default value. 1327 * @throws NumberFormatException when the value is invalid 1328 * @return property value as a <code>long</code>, 1329 * or <code>defaultValue</code>. 1330 */ 1331 public long getLongBytes(String name, long defaultValue) { 1332 String valueString = getTrimmed(name); 1333 if (valueString == null) 1334 return defaultValue; 1335 return StringUtils.TraditionalBinaryPrefix.string2long(valueString); 1336 } 1337 1338 private String getHexDigits(String value) { 1339 boolean negative = false; 1340 String str = value; 1341 String hexString = null; 1342 if (value.startsWith("-")) { 1343 negative = true; 1344 str = value.substring(1); 1345 } 1346 if (str.startsWith("0x") || str.startsWith("0X")) { 1347 hexString = str.substring(2); 1348 if (negative) { 1349 hexString = "-" + hexString; 1350 } 1351 return hexString; 1352 } 1353 return null; 1354 } 1355 1356 /** 1357 * Set the value of the <code>name</code> property to a <code>long</code>. 1358 * 1359 * @param name property name. 1360 * @param value <code>long</code> value of the property. 1361 */ 1362 public void setLong(String name, long value) { 1363 set(name, Long.toString(value)); 1364 } 1365 1366 /** 1367 * Get the value of the <code>name</code> property as a <code>float</code>. 1368 * If no such property exists, the provided default value is returned, 1369 * or if the specified value is not a valid <code>float</code>, 1370 * then an error is thrown. 1371 * 1372 * @param name property name. 1373 * @param defaultValue default value. 1374 * @throws NumberFormatException when the value is invalid 1375 * @return property value as a <code>float</code>, 1376 * or <code>defaultValue</code>. 1377 */ 1378 public float getFloat(String name, float defaultValue) { 1379 String valueString = getTrimmed(name); 1380 if (valueString == null) 1381 return defaultValue; 1382 return Float.parseFloat(valueString); 1383 } 1384 1385 /** 1386 * Set the value of the <code>name</code> property to a <code>float</code>. 1387 * 1388 * @param name property name. 1389 * @param value property value. 1390 */ 1391 public void setFloat(String name, float value) { 1392 set(name,Float.toString(value)); 1393 } 1394 1395 /** 1396 * Get the value of the <code>name</code> property as a <code>double</code>. 1397 * If no such property exists, the provided default value is returned, 1398 * or if the specified value is not a valid <code>double</code>, 1399 * then an error is thrown. 1400 * 1401 * @param name property name. 1402 * @param defaultValue default value. 1403 * @throws NumberFormatException when the value is invalid 1404 * @return property value as a <code>double</code>, 1405 * or <code>defaultValue</code>. 1406 */ 1407 public double getDouble(String name, double defaultValue) { 1408 String valueString = getTrimmed(name); 1409 if (valueString == null) 1410 return defaultValue; 1411 return Double.parseDouble(valueString); 1412 } 1413 1414 /** 1415 * Set the value of the <code>name</code> property to a <code>double</code>. 1416 * 1417 * @param name property name. 1418 * @param value property value. 1419 */ 1420 public void setDouble(String name, double value) { 1421 set(name,Double.toString(value)); 1422 } 1423 1424 /** 1425 * Get the value of the <code>name</code> property as a <code>boolean</code>. 1426 * If no such property is specified, or if the specified value is not a valid 1427 * <code>boolean</code>, then <code>defaultValue</code> is returned. 1428 * 1429 * @param name property name. 1430 * @param defaultValue default value. 1431 * @return property value as a <code>boolean</code>, 1432 * or <code>defaultValue</code>. 1433 */ 1434 public boolean getBoolean(String name, boolean defaultValue) { 1435 String valueString = getTrimmed(name); 1436 if (null == valueString || valueString.isEmpty()) { 1437 return defaultValue; 1438 } 1439 1440 valueString = valueString.toLowerCase(); 1441 1442 if ("true".equals(valueString)) 1443 return true; 1444 else if ("false".equals(valueString)) 1445 return false; 1446 else return defaultValue; 1447 } 1448 1449 /** 1450 * Set the value of the <code>name</code> property to a <code>boolean</code>. 1451 * 1452 * @param name property name. 1453 * @param value <code>boolean</code> value of the property. 1454 */ 1455 public void setBoolean(String name, boolean value) { 1456 set(name, Boolean.toString(value)); 1457 } 1458 1459 /** 1460 * Set the given property, if it is currently unset. 1461 * @param name property name 1462 * @param value new value 1463 */ 1464 public void setBooleanIfUnset(String name, boolean value) { 1465 setIfUnset(name, Boolean.toString(value)); 1466 } 1467 1468 /** 1469 * Set the value of the <code>name</code> property to the given type. This 1470 * is equivalent to <code>set(<name>, value.toString())</code>. 1471 * @param name property name 1472 * @param value new value 1473 */ 1474 public <T extends Enum<T>> void setEnum(String name, T value) { 1475 set(name, value.toString()); 1476 } 1477 1478 /** 1479 * Return value matching this enumerated type. 1480 * @param name Property name 1481 * @param defaultValue Value returned if no mapping exists 1482 * @throws IllegalArgumentException If mapping is illegal for the type 1483 * provided 1484 */ 1485 public <T extends Enum<T>> T getEnum(String name, T defaultValue) { 1486 final String val = get(name); 1487 return null == val 1488 ? defaultValue 1489 : Enum.valueOf(defaultValue.getDeclaringClass(), val); 1490 } 1491 1492 enum ParsedTimeDuration { 1493 NS { 1494 TimeUnit unit() { return TimeUnit.NANOSECONDS; } 1495 String suffix() { return "ns"; } 1496 }, 1497 US { 1498 TimeUnit unit() { return TimeUnit.MICROSECONDS; } 1499 String suffix() { return "us"; } 1500 }, 1501 MS { 1502 TimeUnit unit() { return TimeUnit.MILLISECONDS; } 1503 String suffix() { return "ms"; } 1504 }, 1505 S { 1506 TimeUnit unit() { return TimeUnit.SECONDS; } 1507 String suffix() { return "s"; } 1508 }, 1509 M { 1510 TimeUnit unit() { return TimeUnit.MINUTES; } 1511 String suffix() { return "m"; } 1512 }, 1513 H { 1514 TimeUnit unit() { return TimeUnit.HOURS; } 1515 String suffix() { return "h"; } 1516 }, 1517 D { 1518 TimeUnit unit() { return TimeUnit.DAYS; } 1519 String suffix() { return "d"; } 1520 }; 1521 abstract TimeUnit unit(); 1522 abstract String suffix(); 1523 static ParsedTimeDuration unitFor(String s) { 1524 for (ParsedTimeDuration ptd : values()) { 1525 // iteration order is in decl order, so SECONDS matched last 1526 if (s.endsWith(ptd.suffix())) { 1527 return ptd; 1528 } 1529 } 1530 return null; 1531 } 1532 static ParsedTimeDuration unitFor(TimeUnit unit) { 1533 for (ParsedTimeDuration ptd : values()) { 1534 if (ptd.unit() == unit) { 1535 return ptd; 1536 } 1537 } 1538 return null; 1539 } 1540 } 1541 1542 /** 1543 * Set the value of <code>name</code> to the given time duration. This 1544 * is equivalent to <code>set(<name>, value + <time suffix>)</code>. 1545 * @param name Property name 1546 * @param value Time duration 1547 * @param unit Unit of time 1548 */ 1549 public void setTimeDuration(String name, long value, TimeUnit unit) { 1550 set(name, value + ParsedTimeDuration.unitFor(unit).suffix()); 1551 } 1552 1553 /** 1554 * Return time duration in the given time unit. Valid units are encoded in 1555 * properties as suffixes: nanoseconds (ns), microseconds (us), milliseconds 1556 * (ms), seconds (s), minutes (m), hours (h), and days (d). 1557 * @param name Property name 1558 * @param defaultValue Value returned if no mapping exists. 1559 * @param unit Unit to convert the stored property, if it exists. 1560 * @throws NumberFormatException If the property stripped of its unit is not 1561 * a number 1562 */ 1563 public long getTimeDuration(String name, long defaultValue, TimeUnit unit) { 1564 String vStr = get(name); 1565 if (null == vStr) { 1566 return defaultValue; 1567 } 1568 vStr = vStr.trim(); 1569 ParsedTimeDuration vUnit = ParsedTimeDuration.unitFor(vStr); 1570 if (null == vUnit) { 1571 LOG.warn("No unit for " + name + "(" + vStr + ") assuming " + unit); 1572 vUnit = ParsedTimeDuration.unitFor(unit); 1573 } else { 1574 vStr = vStr.substring(0, vStr.lastIndexOf(vUnit.suffix())); 1575 } 1576 return unit.convert(Long.parseLong(vStr), vUnit.unit()); 1577 } 1578 1579 /** 1580 * Get the value of the <code>name</code> property as a <code>Pattern</code>. 1581 * If no such property is specified, or if the specified value is not a valid 1582 * <code>Pattern</code>, then <code>DefaultValue</code> is returned. 1583 * 1584 * @param name property name 1585 * @param defaultValue default value 1586 * @return property value as a compiled Pattern, or defaultValue 1587 */ 1588 public Pattern getPattern(String name, Pattern defaultValue) { 1589 String valString = get(name); 1590 if (null == valString || valString.isEmpty()) { 1591 return defaultValue; 1592 } 1593 try { 1594 return Pattern.compile(valString); 1595 } catch (PatternSyntaxException pse) { 1596 LOG.warn("Regular expression '" + valString + "' for property '" + 1597 name + "' not valid. Using default", pse); 1598 return defaultValue; 1599 } 1600 } 1601 1602 /** 1603 * Set the given property to <code>Pattern</code>. 1604 * If the pattern is passed as null, sets the empty pattern which results in 1605 * further calls to getPattern(...) returning the default value. 1606 * 1607 * @param name property name 1608 * @param pattern new value 1609 */ 1610 public void setPattern(String name, Pattern pattern) { 1611 if (null == pattern) { 1612 set(name, null); 1613 } else { 1614 set(name, pattern.pattern()); 1615 } 1616 } 1617 1618 /** 1619 * Gets information about why a property was set. Typically this is the 1620 * path to the resource objects (file, URL, etc.) the property came from, but 1621 * it can also indicate that it was set programatically, or because of the 1622 * command line. 1623 * 1624 * @param name - The property name to get the source of. 1625 * @return null - If the property or its source wasn't found. Otherwise, 1626 * returns a list of the sources of the resource. The older sources are 1627 * the first ones in the list. So for example if a configuration is set from 1628 * the command line, and then written out to a file that is read back in the 1629 * first entry would indicate that it was set from the command line, while 1630 * the second one would indicate the file that the new configuration was read 1631 * in from. 1632 */ 1633 @InterfaceStability.Unstable 1634 public synchronized String[] getPropertySources(String name) { 1635 if (properties == null) { 1636 // If properties is null, it means a resource was newly added 1637 // but the props were cleared so as to load it upon future 1638 // requests. So lets force a load by asking a properties list. 1639 getProps(); 1640 } 1641 // Return a null right away if our properties still 1642 // haven't loaded or the resource mapping isn't defined 1643 if (properties == null || updatingResource == null) { 1644 return null; 1645 } else { 1646 String[] source = updatingResource.get(name); 1647 if(source == null) { 1648 return null; 1649 } else { 1650 return Arrays.copyOf(source, source.length); 1651 } 1652 } 1653 } 1654 1655 /** 1656 * A class that represents a set of positive integer ranges. It parses 1657 * strings of the form: "2-3,5,7-" where ranges are separated by comma and 1658 * the lower/upper bounds are separated by dash. Either the lower or upper 1659 * bound may be omitted meaning all values up to or over. So the string 1660 * above means 2, 3, 5, and 7, 8, 9, ... 1661 */ 1662 public static class IntegerRanges implements Iterable<Integer>{ 1663 private static class Range { 1664 int start; 1665 int end; 1666 } 1667 1668 private static class RangeNumberIterator implements Iterator<Integer> { 1669 Iterator<Range> internal; 1670 int at; 1671 int end; 1672 1673 public RangeNumberIterator(List<Range> ranges) { 1674 if (ranges != null) { 1675 internal = ranges.iterator(); 1676 } 1677 at = -1; 1678 end = -2; 1679 } 1680 1681 @Override 1682 public boolean hasNext() { 1683 if (at <= end) { 1684 return true; 1685 } else if (internal != null){ 1686 return internal.hasNext(); 1687 } 1688 return false; 1689 } 1690 1691 @Override 1692 public Integer next() { 1693 if (at <= end) { 1694 at++; 1695 return at - 1; 1696 } else if (internal != null){ 1697 Range found = internal.next(); 1698 if (found != null) { 1699 at = found.start; 1700 end = found.end; 1701 at++; 1702 return at - 1; 1703 } 1704 } 1705 return null; 1706 } 1707 1708 @Override 1709 public void remove() { 1710 throw new UnsupportedOperationException(); 1711 } 1712 }; 1713 1714 List<Range> ranges = new ArrayList<Range>(); 1715 1716 public IntegerRanges() { 1717 } 1718 1719 public IntegerRanges(String newValue) { 1720 StringTokenizer itr = new StringTokenizer(newValue, ","); 1721 while (itr.hasMoreTokens()) { 1722 String rng = itr.nextToken().trim(); 1723 String[] parts = rng.split("-", 3); 1724 if (parts.length < 1 || parts.length > 2) { 1725 throw new IllegalArgumentException("integer range badly formed: " + 1726 rng); 1727 } 1728 Range r = new Range(); 1729 r.start = convertToInt(parts[0], 0); 1730 if (parts.length == 2) { 1731 r.end = convertToInt(parts[1], Integer.MAX_VALUE); 1732 } else { 1733 r.end = r.start; 1734 } 1735 if (r.start > r.end) { 1736 throw new IllegalArgumentException("IntegerRange from " + r.start + 1737 " to " + r.end + " is invalid"); 1738 } 1739 ranges.add(r); 1740 } 1741 } 1742 1743 /** 1744 * Convert a string to an int treating empty strings as the default value. 1745 * @param value the string value 1746 * @param defaultValue the value for if the string is empty 1747 * @return the desired integer 1748 */ 1749 private static int convertToInt(String value, int defaultValue) { 1750 String trim = value.trim(); 1751 if (trim.length() == 0) { 1752 return defaultValue; 1753 } 1754 return Integer.parseInt(trim); 1755 } 1756 1757 /** 1758 * Is the given value in the set of ranges 1759 * @param value the value to check 1760 * @return is the value in the ranges? 1761 */ 1762 public boolean isIncluded(int value) { 1763 for(Range r: ranges) { 1764 if (r.start <= value && value <= r.end) { 1765 return true; 1766 } 1767 } 1768 return false; 1769 } 1770 1771 /** 1772 * @return true if there are no values in this range, else false. 1773 */ 1774 public boolean isEmpty() { 1775 return ranges == null || ranges.isEmpty(); 1776 } 1777 1778 @Override 1779 public String toString() { 1780 StringBuilder result = new StringBuilder(); 1781 boolean first = true; 1782 for(Range r: ranges) { 1783 if (first) { 1784 first = false; 1785 } else { 1786 result.append(','); 1787 } 1788 result.append(r.start); 1789 result.append('-'); 1790 result.append(r.end); 1791 } 1792 return result.toString(); 1793 } 1794 1795 @Override 1796 public Iterator<Integer> iterator() { 1797 return new RangeNumberIterator(ranges); 1798 } 1799 1800 } 1801 1802 /** 1803 * Parse the given attribute as a set of integer ranges 1804 * @param name the attribute name 1805 * @param defaultValue the default value if it is not set 1806 * @return a new set of ranges from the configured value 1807 */ 1808 public IntegerRanges getRange(String name, String defaultValue) { 1809 return new IntegerRanges(get(name, defaultValue)); 1810 } 1811 1812 /** 1813 * Get the comma delimited values of the <code>name</code> property as 1814 * a collection of <code>String</code>s. 1815 * If no such property is specified then empty collection is returned. 1816 * <p> 1817 * This is an optimized version of {@link #getStrings(String)} 1818 * 1819 * @param name property name. 1820 * @return property value as a collection of <code>String</code>s. 1821 */ 1822 public Collection<String> getStringCollection(String name) { 1823 String valueString = get(name); 1824 return StringUtils.getStringCollection(valueString); 1825 } 1826 1827 /** 1828 * Get the comma delimited values of the <code>name</code> property as 1829 * an array of <code>String</code>s. 1830 * If no such property is specified then <code>null</code> is returned. 1831 * 1832 * @param name property name. 1833 * @return property value as an array of <code>String</code>s, 1834 * or <code>null</code>. 1835 */ 1836 public String[] getStrings(String name) { 1837 String valueString = get(name); 1838 return StringUtils.getStrings(valueString); 1839 } 1840 1841 /** 1842 * Get the comma delimited values of the <code>name</code> property as 1843 * an array of <code>String</code>s. 1844 * If no such property is specified then default value is returned. 1845 * 1846 * @param name property name. 1847 * @param defaultValue The default value 1848 * @return property value as an array of <code>String</code>s, 1849 * or default value. 1850 */ 1851 public String[] getStrings(String name, String... defaultValue) { 1852 String valueString = get(name); 1853 if (valueString == null) { 1854 return defaultValue; 1855 } else { 1856 return StringUtils.getStrings(valueString); 1857 } 1858 } 1859 1860 /** 1861 * Get the comma delimited values of the <code>name</code> property as 1862 * a collection of <code>String</code>s, trimmed of the leading and trailing whitespace. 1863 * If no such property is specified then empty <code>Collection</code> is returned. 1864 * 1865 * @param name property name. 1866 * @return property value as a collection of <code>String</code>s, or empty <code>Collection</code> 1867 */ 1868 public Collection<String> getTrimmedStringCollection(String name) { 1869 String valueString = get(name); 1870 if (null == valueString) { 1871 Collection<String> empty = new ArrayList<String>(); 1872 return empty; 1873 } 1874 return StringUtils.getTrimmedStringCollection(valueString); 1875 } 1876 1877 /** 1878 * Get the comma delimited values of the <code>name</code> property as 1879 * an array of <code>String</code>s, trimmed of the leading and trailing whitespace. 1880 * If no such property is specified then an empty array is returned. 1881 * 1882 * @param name property name. 1883 * @return property value as an array of trimmed <code>String</code>s, 1884 * or empty array. 1885 */ 1886 public String[] getTrimmedStrings(String name) { 1887 String valueString = get(name); 1888 return StringUtils.getTrimmedStrings(valueString); 1889 } 1890 1891 /** 1892 * Get the comma delimited values of the <code>name</code> property as 1893 * an array of <code>String</code>s, trimmed of the leading and trailing whitespace. 1894 * If no such property is specified then default value is returned. 1895 * 1896 * @param name property name. 1897 * @param defaultValue The default value 1898 * @return property value as an array of trimmed <code>String</code>s, 1899 * or default value. 1900 */ 1901 public String[] getTrimmedStrings(String name, String... defaultValue) { 1902 String valueString = get(name); 1903 if (null == valueString) { 1904 return defaultValue; 1905 } else { 1906 return StringUtils.getTrimmedStrings(valueString); 1907 } 1908 } 1909 1910 /** 1911 * Set the array of string values for the <code>name</code> property as 1912 * as comma delimited values. 1913 * 1914 * @param name property name. 1915 * @param values The values 1916 */ 1917 public void setStrings(String name, String... values) { 1918 set(name, StringUtils.arrayToString(values)); 1919 } 1920 1921 /** 1922 * Get the value for a known password configuration element. 1923 * In order to enable the elimination of clear text passwords in config, 1924 * this method attempts to resolve the property name as an alias through 1925 * the CredentialProvider API and conditionally fallsback to config. 1926 * @param name property name 1927 * @return password 1928 */ 1929 public char[] getPassword(String name) throws IOException { 1930 char[] pass = null; 1931 1932 pass = getPasswordFromCredentialProviders(name); 1933 1934 if (pass == null) { 1935 pass = getPasswordFromConfig(name); 1936 } 1937 1938 return pass; 1939 } 1940 1941 /** 1942 * Try and resolve the provided element name as a credential provider 1943 * alias. 1944 * @param name alias of the provisioned credential 1945 * @return password or null if not found 1946 * @throws IOException 1947 */ 1948 protected char[] getPasswordFromCredentialProviders(String name) 1949 throws IOException { 1950 char[] pass = null; 1951 try { 1952 List<CredentialProvider> providers = 1953 CredentialProviderFactory.getProviders(this); 1954 1955 if (providers != null) { 1956 for (CredentialProvider provider : providers) { 1957 try { 1958 CredentialEntry entry = provider.getCredentialEntry(name); 1959 if (entry != null) { 1960 pass = entry.getCredential(); 1961 break; 1962 } 1963 } 1964 catch (IOException ioe) { 1965 throw new IOException("Can't get key " + name + " from key provider" + 1966 "of type: " + provider.getClass().getName() + ".", ioe); 1967 } 1968 } 1969 } 1970 } 1971 catch (IOException ioe) { 1972 throw new IOException("Configuration problem with provider path.", ioe); 1973 } 1974 1975 return pass; 1976 } 1977 1978 /** 1979 * Fallback to clear text passwords in configuration. 1980 * @param name 1981 * @return clear text password or null 1982 */ 1983 protected char[] getPasswordFromConfig(String name) { 1984 char[] pass = null; 1985 if (getBoolean(CredentialProvider.CLEAR_TEXT_FALLBACK, 1986 CommonConfigurationKeysPublic. 1987 HADOOP_SECURITY_CREDENTIAL_CLEAR_TEXT_FALLBACK_DEFAULT)) { 1988 String passStr = get(name); 1989 if (passStr != null) { 1990 pass = passStr.toCharArray(); 1991 } 1992 } 1993 return pass; 1994 } 1995 1996 /** 1997 * Get the socket address for <code>hostProperty</code> as a 1998 * <code>InetSocketAddress</code>. If <code>hostProperty</code> is 1999 * <code>null</code>, <code>addressProperty</code> will be used. This 2000 * is useful for cases where we want to differentiate between host 2001 * bind address and address clients should use to establish connection. 2002 * 2003 * @param hostProperty bind host property name. 2004 * @param addressProperty address property name. 2005 * @param defaultAddressValue the default value 2006 * @param defaultPort the default port 2007 * @return InetSocketAddress 2008 */ 2009 public InetSocketAddress getSocketAddr( 2010 String hostProperty, 2011 String addressProperty, 2012 String defaultAddressValue, 2013 int defaultPort) { 2014 2015 InetSocketAddress bindAddr = getSocketAddr( 2016 addressProperty, defaultAddressValue, defaultPort); 2017 2018 final String host = get(hostProperty); 2019 2020 if (host == null || host.isEmpty()) { 2021 return bindAddr; 2022 } 2023 2024 return NetUtils.createSocketAddr( 2025 host, bindAddr.getPort(), hostProperty); 2026 } 2027 2028 /** 2029 * Get the socket address for <code>name</code> property as a 2030 * <code>InetSocketAddress</code>. 2031 * @param name property name. 2032 * @param defaultAddress the default value 2033 * @param defaultPort the default port 2034 * @return InetSocketAddress 2035 */ 2036 public InetSocketAddress getSocketAddr( 2037 String name, String defaultAddress, int defaultPort) { 2038 final String address = get(name, defaultAddress); 2039 return NetUtils.createSocketAddr(address, defaultPort, name); 2040 } 2041 2042 /** 2043 * Set the socket address for the <code>name</code> property as 2044 * a <code>host:port</code>. 2045 */ 2046 public void setSocketAddr(String name, InetSocketAddress addr) { 2047 set(name, NetUtils.getHostPortString(addr)); 2048 } 2049 2050 /** 2051 * Set the socket address a client can use to connect for the 2052 * <code>name</code> property as a <code>host:port</code>. The wildcard 2053 * address is replaced with the local host's address. If the host and address 2054 * properties are configured the host component of the address will be combined 2055 * with the port component of the addr to generate the address. This is to allow 2056 * optional control over which host name is used in multi-home bind-host 2057 * cases where a host can have multiple names 2058 * @param hostProperty the bind-host configuration name 2059 * @param addressProperty the service address configuration name 2060 * @param defaultAddressValue the service default address configuration value 2061 * @param addr InetSocketAddress of the service listener 2062 * @return InetSocketAddress for clients to connect 2063 */ 2064 public InetSocketAddress updateConnectAddr( 2065 String hostProperty, 2066 String addressProperty, 2067 String defaultAddressValue, 2068 InetSocketAddress addr) { 2069 2070 final String host = get(hostProperty); 2071 final String connectHostPort = getTrimmed(addressProperty, defaultAddressValue); 2072 2073 if (host == null || host.isEmpty() || connectHostPort == null || connectHostPort.isEmpty()) { 2074 //not our case, fall back to original logic 2075 return updateConnectAddr(addressProperty, addr); 2076 } 2077 2078 final String connectHost = connectHostPort.split(":")[0]; 2079 // Create connect address using client address hostname and server port. 2080 return updateConnectAddr(addressProperty, NetUtils.createSocketAddrForHost( 2081 connectHost, addr.getPort())); 2082 } 2083 2084 /** 2085 * Set the socket address a client can use to connect for the 2086 * <code>name</code> property as a <code>host:port</code>. The wildcard 2087 * address is replaced with the local host's address. 2088 * @param name property name. 2089 * @param addr InetSocketAddress of a listener to store in the given property 2090 * @return InetSocketAddress for clients to connect 2091 */ 2092 public InetSocketAddress updateConnectAddr(String name, 2093 InetSocketAddress addr) { 2094 final InetSocketAddress connectAddr = NetUtils.getConnectAddress(addr); 2095 setSocketAddr(name, connectAddr); 2096 return connectAddr; 2097 } 2098 2099 /** 2100 * Load a class by name. 2101 * 2102 * @param name the class name. 2103 * @return the class object. 2104 * @throws ClassNotFoundException if the class is not found. 2105 */ 2106 public Class<?> getClassByName(String name) throws ClassNotFoundException { 2107 Class<?> ret = getClassByNameOrNull(name); 2108 if (ret == null) { 2109 throw new ClassNotFoundException("Class " + name + " not found"); 2110 } 2111 return ret; 2112 } 2113 2114 /** 2115 * Load a class by name, returning null rather than throwing an exception 2116 * if it couldn't be loaded. This is to avoid the overhead of creating 2117 * an exception. 2118 * 2119 * @param name the class name 2120 * @return the class object, or null if it could not be found. 2121 */ 2122 public Class<?> getClassByNameOrNull(String name) { 2123 Map<String, WeakReference<Class<?>>> map; 2124 2125 synchronized (CACHE_CLASSES) { 2126 map = CACHE_CLASSES.get(classLoader); 2127 if (map == null) { 2128 map = Collections.synchronizedMap( 2129 new WeakHashMap<String, WeakReference<Class<?>>>()); 2130 CACHE_CLASSES.put(classLoader, map); 2131 } 2132 } 2133 2134 Class<?> clazz = null; 2135 WeakReference<Class<?>> ref = map.get(name); 2136 if (ref != null) { 2137 clazz = ref.get(); 2138 } 2139 2140 if (clazz == null) { 2141 try { 2142 clazz = Class.forName(name, true, classLoader); 2143 } catch (ClassNotFoundException e) { 2144 // Leave a marker that the class isn't found 2145 map.put(name, new WeakReference<Class<?>>(NEGATIVE_CACHE_SENTINEL)); 2146 return null; 2147 } 2148 // two putters can race here, but they'll put the same class 2149 map.put(name, new WeakReference<Class<?>>(clazz)); 2150 return clazz; 2151 } else if (clazz == NEGATIVE_CACHE_SENTINEL) { 2152 return null; // not found 2153 } else { 2154 // cache hit 2155 return clazz; 2156 } 2157 } 2158 2159 /** 2160 * Get the value of the <code>name</code> property 2161 * as an array of <code>Class</code>. 2162 * The value of the property specifies a list of comma separated class names. 2163 * If no such property is specified, then <code>defaultValue</code> is 2164 * returned. 2165 * 2166 * @param name the property name. 2167 * @param defaultValue default value. 2168 * @return property value as a <code>Class[]</code>, 2169 * or <code>defaultValue</code>. 2170 */ 2171 public Class<?>[] getClasses(String name, Class<?> ... defaultValue) { 2172 String[] classnames = getTrimmedStrings(name); 2173 if (classnames == null) 2174 return defaultValue; 2175 try { 2176 Class<?>[] classes = new Class<?>[classnames.length]; 2177 for(int i = 0; i < classnames.length; i++) { 2178 classes[i] = getClassByName(classnames[i]); 2179 } 2180 return classes; 2181 } catch (ClassNotFoundException e) { 2182 throw new RuntimeException(e); 2183 } 2184 } 2185 2186 /** 2187 * Get the value of the <code>name</code> property as a <code>Class</code>. 2188 * If no such property is specified, then <code>defaultValue</code> is 2189 * returned. 2190 * 2191 * @param name the class name. 2192 * @param defaultValue default value. 2193 * @return property value as a <code>Class</code>, 2194 * or <code>defaultValue</code>. 2195 */ 2196 public Class<?> getClass(String name, Class<?> defaultValue) { 2197 String valueString = getTrimmed(name); 2198 if (valueString == null) 2199 return defaultValue; 2200 try { 2201 return getClassByName(valueString); 2202 } catch (ClassNotFoundException e) { 2203 throw new RuntimeException(e); 2204 } 2205 } 2206 2207 /** 2208 * Get the value of the <code>name</code> property as a <code>Class</code> 2209 * implementing the interface specified by <code>xface</code>. 2210 * 2211 * If no such property is specified, then <code>defaultValue</code> is 2212 * returned. 2213 * 2214 * An exception is thrown if the returned class does not implement the named 2215 * interface. 2216 * 2217 * @param name the class name. 2218 * @param defaultValue default value. 2219 * @param xface the interface implemented by the named class. 2220 * @return property value as a <code>Class</code>, 2221 * or <code>defaultValue</code>. 2222 */ 2223 public <U> Class<? extends U> getClass(String name, 2224 Class<? extends U> defaultValue, 2225 Class<U> xface) { 2226 try { 2227 Class<?> theClass = getClass(name, defaultValue); 2228 if (theClass != null && !xface.isAssignableFrom(theClass)) 2229 throw new RuntimeException(theClass+" not "+xface.getName()); 2230 else if (theClass != null) 2231 return theClass.asSubclass(xface); 2232 else 2233 return null; 2234 } catch (Exception e) { 2235 throw new RuntimeException(e); 2236 } 2237 } 2238 2239 /** 2240 * Get the value of the <code>name</code> property as a <code>List</code> 2241 * of objects implementing the interface specified by <code>xface</code>. 2242 * 2243 * An exception is thrown if any of the classes does not exist, or if it does 2244 * not implement the named interface. 2245 * 2246 * @param name the property name. 2247 * @param xface the interface implemented by the classes named by 2248 * <code>name</code>. 2249 * @return a <code>List</code> of objects implementing <code>xface</code>. 2250 */ 2251 @SuppressWarnings("unchecked") 2252 public <U> List<U> getInstances(String name, Class<U> xface) { 2253 List<U> ret = new ArrayList<U>(); 2254 Class<?>[] classes = getClasses(name); 2255 for (Class<?> cl: classes) { 2256 if (!xface.isAssignableFrom(cl)) { 2257 throw new RuntimeException(cl + " does not implement " + xface); 2258 } 2259 ret.add((U)ReflectionUtils.newInstance(cl, this)); 2260 } 2261 return ret; 2262 } 2263 2264 /** 2265 * Set the value of the <code>name</code> property to the name of a 2266 * <code>theClass</code> implementing the given interface <code>xface</code>. 2267 * 2268 * An exception is thrown if <code>theClass</code> does not implement the 2269 * interface <code>xface</code>. 2270 * 2271 * @param name property name. 2272 * @param theClass property value. 2273 * @param xface the interface implemented by the named class. 2274 */ 2275 public void setClass(String name, Class<?> theClass, Class<?> xface) { 2276 if (!xface.isAssignableFrom(theClass)) 2277 throw new RuntimeException(theClass+" not "+xface.getName()); 2278 set(name, theClass.getName()); 2279 } 2280 2281 /** 2282 * Get a local file under a directory named by <i>dirsProp</i> with 2283 * the given <i>path</i>. If <i>dirsProp</i> contains multiple directories, 2284 * then one is chosen based on <i>path</i>'s hash code. If the selected 2285 * directory does not exist, an attempt is made to create it. 2286 * 2287 * @param dirsProp directory in which to locate the file. 2288 * @param path file-path. 2289 * @return local file under the directory with the given path. 2290 */ 2291 public Path getLocalPath(String dirsProp, String path) 2292 throws IOException { 2293 String[] dirs = getTrimmedStrings(dirsProp); 2294 int hashCode = path.hashCode(); 2295 FileSystem fs = FileSystem.getLocal(this); 2296 for (int i = 0; i < dirs.length; i++) { // try each local dir 2297 int index = (hashCode+i & Integer.MAX_VALUE) % dirs.length; 2298 Path file = new Path(dirs[index], path); 2299 Path dir = file.getParent(); 2300 if (fs.mkdirs(dir) || fs.exists(dir)) { 2301 return file; 2302 } 2303 } 2304 LOG.warn("Could not make " + path + 2305 " in local directories from " + dirsProp); 2306 for(int i=0; i < dirs.length; i++) { 2307 int index = (hashCode+i & Integer.MAX_VALUE) % dirs.length; 2308 LOG.warn(dirsProp + "[" + index + "]=" + dirs[index]); 2309 } 2310 throw new IOException("No valid local directories in property: "+dirsProp); 2311 } 2312 2313 /** 2314 * Get a local file name under a directory named in <i>dirsProp</i> with 2315 * the given <i>path</i>. If <i>dirsProp</i> contains multiple directories, 2316 * then one is chosen based on <i>path</i>'s hash code. If the selected 2317 * directory does not exist, an attempt is made to create it. 2318 * 2319 * @param dirsProp directory in which to locate the file. 2320 * @param path file-path. 2321 * @return local file under the directory with the given path. 2322 */ 2323 public File getFile(String dirsProp, String path) 2324 throws IOException { 2325 String[] dirs = getTrimmedStrings(dirsProp); 2326 int hashCode = path.hashCode(); 2327 for (int i = 0; i < dirs.length; i++) { // try each local dir 2328 int index = (hashCode+i & Integer.MAX_VALUE) % dirs.length; 2329 File file = new File(dirs[index], path); 2330 File dir = file.getParentFile(); 2331 if (dir.exists() || dir.mkdirs()) { 2332 return file; 2333 } 2334 } 2335 throw new IOException("No valid local directories in property: "+dirsProp); 2336 } 2337 2338 /** 2339 * Get the {@link URL} for the named resource. 2340 * 2341 * @param name resource name. 2342 * @return the url for the named resource. 2343 */ 2344 public URL getResource(String name) { 2345 return classLoader.getResource(name); 2346 } 2347 2348 /** 2349 * Get an input stream attached to the configuration resource with the 2350 * given <code>name</code>. 2351 * 2352 * @param name configuration resource name. 2353 * @return an input stream attached to the resource. 2354 */ 2355 public InputStream getConfResourceAsInputStream(String name) { 2356 try { 2357 URL url= getResource(name); 2358 2359 if (url == null) { 2360 LOG.info(name + " not found"); 2361 return null; 2362 } else { 2363 LOG.info("found resource " + name + " at " + url); 2364 } 2365 2366 return url.openStream(); 2367 } catch (Exception e) { 2368 return null; 2369 } 2370 } 2371 2372 /** 2373 * Get a {@link Reader} attached to the configuration resource with the 2374 * given <code>name</code>. 2375 * 2376 * @param name configuration resource name. 2377 * @return a reader attached to the resource. 2378 */ 2379 public Reader getConfResourceAsReader(String name) { 2380 try { 2381 URL url= getResource(name); 2382 2383 if (url == null) { 2384 LOG.info(name + " not found"); 2385 return null; 2386 } else { 2387 LOG.info("found resource " + name + " at " + url); 2388 } 2389 2390 return new InputStreamReader(url.openStream()); 2391 } catch (Exception e) { 2392 return null; 2393 } 2394 } 2395 2396 /** 2397 * Get the set of parameters marked final. 2398 * 2399 * @return final parameter set. 2400 */ 2401 public Set<String> getFinalParameters() { 2402 Set<String> setFinalParams = Collections.newSetFromMap( 2403 new ConcurrentHashMap<String, Boolean>()); 2404 setFinalParams.addAll(finalParameters); 2405 return setFinalParams; 2406 } 2407 2408 protected synchronized Properties getProps() { 2409 if (properties == null) { 2410 properties = new Properties(); 2411 Map<String, String[]> backup = 2412 new ConcurrentHashMap<String, String[]>(updatingResource); 2413 loadResources(properties, resources, quietmode); 2414 2415 if (overlay != null) { 2416 properties.putAll(overlay); 2417 for (Map.Entry<Object,Object> item: overlay.entrySet()) { 2418 String key = (String)item.getKey(); 2419 String[] source = backup.get(key); 2420 if(source != null) { 2421 updatingResource.put(key, source); 2422 } 2423 } 2424 } 2425 } 2426 return properties; 2427 } 2428 2429 /** 2430 * Return the number of keys in the configuration. 2431 * 2432 * @return number of keys in the configuration. 2433 */ 2434 public int size() { 2435 return getProps().size(); 2436 } 2437 2438 /** 2439 * Clears all keys from the configuration. 2440 */ 2441 public void clear() { 2442 getProps().clear(); 2443 getOverlay().clear(); 2444 } 2445 2446 /** 2447 * Get an {@link Iterator} to go through the list of <code>String</code> 2448 * key-value pairs in the configuration. 2449 * 2450 * @return an iterator over the entries. 2451 */ 2452 @Override 2453 public Iterator<Map.Entry<String, String>> iterator() { 2454 // Get a copy of just the string to string pairs. After the old object 2455 // methods that allow non-strings to be put into configurations are removed, 2456 // we could replace properties with a Map<String,String> and get rid of this 2457 // code. 2458 Map<String,String> result = new HashMap<String,String>(); 2459 for(Map.Entry<Object,Object> item: getProps().entrySet()) { 2460 if (item.getKey() instanceof String && 2461 item.getValue() instanceof String) { 2462 result.put((String) item.getKey(), (String) item.getValue()); 2463 } 2464 } 2465 return result.entrySet().iterator(); 2466 } 2467 2468 private Document parse(DocumentBuilder builder, URL url) 2469 throws IOException, SAXException { 2470 if (!quietmode) { 2471 LOG.debug("parsing URL " + url); 2472 } 2473 if (url == null) { 2474 return null; 2475 } 2476 2477 URLConnection connection = url.openConnection(); 2478 if (connection instanceof JarURLConnection) { 2479 // Disable caching for JarURLConnection to avoid sharing JarFile 2480 // with other users. 2481 connection.setUseCaches(false); 2482 } 2483 return parse(builder, connection.getInputStream(), url.toString()); 2484 } 2485 2486 private Document parse(DocumentBuilder builder, InputStream is, 2487 String systemId) throws IOException, SAXException { 2488 if (!quietmode) { 2489 LOG.debug("parsing input stream " + is); 2490 } 2491 if (is == null) { 2492 return null; 2493 } 2494 try { 2495 return (systemId == null) ? builder.parse(is) : builder.parse(is, 2496 systemId); 2497 } finally { 2498 is.close(); 2499 } 2500 } 2501 2502 private void loadResources(Properties properties, 2503 ArrayList<Resource> resources, 2504 boolean quiet) { 2505 if(loadDefaults) { 2506 for (String resource : defaultResources) { 2507 loadResource(properties, new Resource(resource), quiet); 2508 } 2509 2510 //support the hadoop-site.xml as a deprecated case 2511 if(getResource("hadoop-site.xml")!=null) { 2512 loadResource(properties, new Resource("hadoop-site.xml"), quiet); 2513 } 2514 } 2515 2516 for (int i = 0; i < resources.size(); i++) { 2517 Resource ret = loadResource(properties, resources.get(i), quiet); 2518 if (ret != null) { 2519 resources.set(i, ret); 2520 } 2521 } 2522 } 2523 2524 private Resource loadResource(Properties properties, Resource wrapper, boolean quiet) { 2525 String name = UNKNOWN_RESOURCE; 2526 try { 2527 Object resource = wrapper.getResource(); 2528 name = wrapper.getName(); 2529 2530 DocumentBuilderFactory docBuilderFactory 2531 = DocumentBuilderFactory.newInstance(); 2532 //ignore all comments inside the xml file 2533 docBuilderFactory.setIgnoringComments(true); 2534 2535 //allow includes in the xml file 2536 docBuilderFactory.setNamespaceAware(true); 2537 try { 2538 docBuilderFactory.setXIncludeAware(true); 2539 } catch (UnsupportedOperationException e) { 2540 LOG.error("Failed to set setXIncludeAware(true) for parser " 2541 + docBuilderFactory 2542 + ":" + e, 2543 e); 2544 } 2545 DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); 2546 Document doc = null; 2547 Element root = null; 2548 boolean returnCachedProperties = false; 2549 2550 if (resource instanceof URL) { // an URL resource 2551 doc = parse(builder, (URL)resource); 2552 } else if (resource instanceof String) { // a CLASSPATH resource 2553 URL url = getResource((String)resource); 2554 doc = parse(builder, url); 2555 } else if (resource instanceof Path) { // a file resource 2556 // Can't use FileSystem API or we get an infinite loop 2557 // since FileSystem uses Configuration API. Use java.io.File instead. 2558 File file = new File(((Path)resource).toUri().getPath()) 2559 .getAbsoluteFile(); 2560 if (file.exists()) { 2561 if (!quiet) { 2562 LOG.debug("parsing File " + file); 2563 } 2564 doc = parse(builder, new BufferedInputStream( 2565 new FileInputStream(file)), ((Path)resource).toString()); 2566 } 2567 } else if (resource instanceof InputStream) { 2568 doc = parse(builder, (InputStream) resource, null); 2569 returnCachedProperties = true; 2570 } else if (resource instanceof Properties) { 2571 overlay(properties, (Properties)resource); 2572 } else if (resource instanceof Element) { 2573 root = (Element)resource; 2574 } 2575 2576 if (root == null) { 2577 if (doc == null) { 2578 if (quiet) { 2579 return null; 2580 } 2581 throw new RuntimeException(resource + " not found"); 2582 } 2583 root = doc.getDocumentElement(); 2584 } 2585 Properties toAddTo = properties; 2586 if(returnCachedProperties) { 2587 toAddTo = new Properties(); 2588 } 2589 if (!"configuration".equals(root.getTagName())) 2590 LOG.fatal("bad conf file: top-level element not <configuration>"); 2591 NodeList props = root.getChildNodes(); 2592 DeprecationContext deprecations = deprecationContext.get(); 2593 for (int i = 0; i < props.getLength(); i++) { 2594 Node propNode = props.item(i); 2595 if (!(propNode instanceof Element)) 2596 continue; 2597 Element prop = (Element)propNode; 2598 if ("configuration".equals(prop.getTagName())) { 2599 loadResource(toAddTo, new Resource(prop, name), quiet); 2600 continue; 2601 } 2602 if (!"property".equals(prop.getTagName())) 2603 LOG.warn("bad conf file: element not <property>"); 2604 NodeList fields = prop.getChildNodes(); 2605 String attr = null; 2606 String value = null; 2607 boolean finalParameter = false; 2608 LinkedList<String> source = new LinkedList<String>(); 2609 for (int j = 0; j < fields.getLength(); j++) { 2610 Node fieldNode = fields.item(j); 2611 if (!(fieldNode instanceof Element)) 2612 continue; 2613 Element field = (Element)fieldNode; 2614 if ("name".equals(field.getTagName()) && field.hasChildNodes()) 2615 attr = StringInterner.weakIntern( 2616 ((Text)field.getFirstChild()).getData().trim()); 2617 if ("value".equals(field.getTagName()) && field.hasChildNodes()) 2618 value = StringInterner.weakIntern( 2619 ((Text)field.getFirstChild()).getData()); 2620 if ("final".equals(field.getTagName()) && field.hasChildNodes()) 2621 finalParameter = "true".equals(((Text)field.getFirstChild()).getData()); 2622 if ("source".equals(field.getTagName()) && field.hasChildNodes()) 2623 source.add(StringInterner.weakIntern( 2624 ((Text)field.getFirstChild()).getData())); 2625 } 2626 source.add(name); 2627 2628 // Ignore this parameter if it has already been marked as 'final' 2629 if (attr != null) { 2630 if (deprecations.getDeprecatedKeyMap().containsKey(attr)) { 2631 DeprecatedKeyInfo keyInfo = 2632 deprecations.getDeprecatedKeyMap().get(attr); 2633 keyInfo.clearAccessed(); 2634 for (String key:keyInfo.newKeys) { 2635 // update new keys with deprecated key's value 2636 loadProperty(toAddTo, name, key, value, finalParameter, 2637 source.toArray(new String[source.size()])); 2638 } 2639 } 2640 else { 2641 loadProperty(toAddTo, name, attr, value, finalParameter, 2642 source.toArray(new String[source.size()])); 2643 } 2644 } 2645 } 2646 2647 if (returnCachedProperties) { 2648 overlay(properties, toAddTo); 2649 return new Resource(toAddTo, name); 2650 } 2651 return null; 2652 } catch (IOException e) { 2653 LOG.fatal("error parsing conf " + name, e); 2654 throw new RuntimeException(e); 2655 } catch (DOMException e) { 2656 LOG.fatal("error parsing conf " + name, e); 2657 throw new RuntimeException(e); 2658 } catch (SAXException e) { 2659 LOG.fatal("error parsing conf " + name, e); 2660 throw new RuntimeException(e); 2661 } catch (ParserConfigurationException e) { 2662 LOG.fatal("error parsing conf " + name , e); 2663 throw new RuntimeException(e); 2664 } 2665 } 2666 2667 private void overlay(Properties to, Properties from) { 2668 for (Entry<Object, Object> entry: from.entrySet()) { 2669 to.put(entry.getKey(), entry.getValue()); 2670 } 2671 } 2672 2673 private void loadProperty(Properties properties, String name, String attr, 2674 String value, boolean finalParameter, String[] source) { 2675 if (value != null || allowNullValueProperties) { 2676 if (!finalParameters.contains(attr)) { 2677 if (value==null && allowNullValueProperties) { 2678 value = DEFAULT_STRING_CHECK; 2679 } 2680 properties.setProperty(attr, value); 2681 if(source != null) { 2682 updatingResource.put(attr, source); 2683 } 2684 } else if (!value.equals(properties.getProperty(attr))) { 2685 LOG.warn(name+":an attempt to override final parameter: "+attr 2686 +"; Ignoring."); 2687 } 2688 } 2689 if (finalParameter && attr != null) { 2690 finalParameters.add(attr); 2691 } 2692 } 2693 2694 /** 2695 * Write out the non-default properties in this configuration to the given 2696 * {@link OutputStream} using UTF-8 encoding. 2697 * 2698 * @param out the output stream to write to. 2699 */ 2700 public void writeXml(OutputStream out) throws IOException { 2701 writeXml(new OutputStreamWriter(out, "UTF-8")); 2702 } 2703 2704 public void writeXml(Writer out) throws IOException { 2705 writeXml(null, out); 2706 } 2707 2708 /** 2709 * Write out the non-default properties in this configuration to the 2710 * given {@link Writer}. 2711 * 2712 * <li> 2713 * When property name is not empty and the property exists in the 2714 * configuration, this method writes the property and its attributes 2715 * to the {@link Writer}. 2716 * </li> 2717 * <p> 2718 * 2719 * <li> 2720 * When property name is null or empty, this method writes all the 2721 * configuration properties and their attributes to the {@link Writer}. 2722 * </li> 2723 * <p> 2724 * 2725 * <li> 2726 * When property name is not empty but the property doesn't exist in 2727 * the configuration, this method throws an {@link IllegalArgumentException}. 2728 * </li> 2729 * <p> 2730 * @param out the writer to write to. 2731 */ 2732 public void writeXml(String propertyName, Writer out) 2733 throws IOException, IllegalArgumentException { 2734 Document doc = asXmlDocument(propertyName); 2735 2736 try { 2737 DOMSource source = new DOMSource(doc); 2738 StreamResult result = new StreamResult(out); 2739 TransformerFactory transFactory = TransformerFactory.newInstance(); 2740 Transformer transformer = transFactory.newTransformer(); 2741 2742 // Important to not hold Configuration log while writing result, since 2743 // 'out' may be an HDFS stream which needs to lock this configuration 2744 // from another thread. 2745 transformer.transform(source, result); 2746 } catch (TransformerException te) { 2747 throw new IOException(te); 2748 } 2749 } 2750 2751 /** 2752 * Return the XML DOM corresponding to this Configuration. 2753 */ 2754 private synchronized Document asXmlDocument(String propertyName) 2755 throws IOException, IllegalArgumentException { 2756 Document doc; 2757 try { 2758 doc = DocumentBuilderFactory 2759 .newInstance() 2760 .newDocumentBuilder() 2761 .newDocument(); 2762 } catch (ParserConfigurationException pe) { 2763 throw new IOException(pe); 2764 } 2765 2766 Element conf = doc.createElement("configuration"); 2767 doc.appendChild(conf); 2768 conf.appendChild(doc.createTextNode("\n")); 2769 handleDeprecation(); //ensure properties is set and deprecation is handled 2770 2771 if(!Strings.isNullOrEmpty(propertyName)) { 2772 if (!properties.containsKey(propertyName)) { 2773 // given property not found, illegal argument 2774 throw new IllegalArgumentException("Property " + 2775 propertyName + " not found"); 2776 } else { 2777 // given property is found, write single property 2778 appendXMLProperty(doc, conf, propertyName); 2779 conf.appendChild(doc.createTextNode("\n")); 2780 } 2781 } else { 2782 // append all elements 2783 for (Enumeration<Object> e = properties.keys(); e.hasMoreElements();) { 2784 appendXMLProperty(doc, conf, (String)e.nextElement()); 2785 conf.appendChild(doc.createTextNode("\n")); 2786 } 2787 } 2788 return doc; 2789 } 2790 2791 /** 2792 * Append a property with its attributes to a given {#link Document} 2793 * if the property is found in configuration. 2794 * 2795 * @param doc 2796 * @param conf 2797 * @param propertyName 2798 */ 2799 private synchronized void appendXMLProperty(Document doc, Element conf, 2800 String propertyName) { 2801 // skip writing if given property name is empty or null 2802 if (!Strings.isNullOrEmpty(propertyName)) { 2803 String value = properties.getProperty(propertyName); 2804 if (value != null) { 2805 Element propNode = doc.createElement("property"); 2806 conf.appendChild(propNode); 2807 2808 Element nameNode = doc.createElement("name"); 2809 nameNode.appendChild(doc.createTextNode(propertyName)); 2810 propNode.appendChild(nameNode); 2811 2812 Element valueNode = doc.createElement("value"); 2813 valueNode.appendChild(doc.createTextNode( 2814 properties.getProperty(propertyName))); 2815 propNode.appendChild(valueNode); 2816 2817 Element finalNode = doc.createElement("final"); 2818 finalNode.appendChild(doc.createTextNode( 2819 String.valueOf(finalParameters.contains(propertyName)))); 2820 propNode.appendChild(finalNode); 2821 2822 if (updatingResource != null) { 2823 String[] sources = updatingResource.get(propertyName); 2824 if(sources != null) { 2825 for(String s : sources) { 2826 Element sourceNode = doc.createElement("source"); 2827 sourceNode.appendChild(doc.createTextNode(s)); 2828 propNode.appendChild(sourceNode); 2829 } 2830 } 2831 } 2832 } 2833 } 2834 } 2835 2836 /** 2837 * Writes properties and their attributes (final and resource) 2838 * to the given {@link Writer}. 2839 * 2840 * <li> 2841 * When propertyName is not empty, and the property exists 2842 * in the configuration, the format of the output would be, 2843 * <pre> 2844 * { 2845 * "property": { 2846 * "key" : "key1", 2847 * "value" : "value1", 2848 * "isFinal" : "key1.isFinal", 2849 * "resource" : "key1.resource" 2850 * } 2851 * } 2852 * </pre> 2853 * </li> 2854 * 2855 * <li> 2856 * When propertyName is null or empty, it behaves same as 2857 * {@link #dumpConfiguration(Configuration, Writer)}, the 2858 * output would be, 2859 * <pre> 2860 * { "properties" : 2861 * [ { key : "key1", 2862 * value : "value1", 2863 * isFinal : "key1.isFinal", 2864 * resource : "key1.resource" }, 2865 * { key : "key2", 2866 * value : "value2", 2867 * isFinal : "ke2.isFinal", 2868 * resource : "key2.resource" } 2869 * ] 2870 * } 2871 * </pre> 2872 * </li> 2873 * 2874 * <li> 2875 * When propertyName is not empty, and the property is not 2876 * found in the configuration, this method will throw an 2877 * {@link IllegalArgumentException}. 2878 * </li> 2879 * <p> 2880 * @param config the configuration 2881 * @param propertyName property name 2882 * @param out the Writer to write to 2883 * @throws IOException 2884 * @throws IllegalArgumentException when property name is not 2885 * empty and the property is not found in configuration 2886 **/ 2887 public static void dumpConfiguration(Configuration config, 2888 String propertyName, Writer out) throws IOException { 2889 if(Strings.isNullOrEmpty(propertyName)) { 2890 dumpConfiguration(config, out); 2891 } else if (Strings.isNullOrEmpty(config.get(propertyName))) { 2892 throw new IllegalArgumentException("Property " + 2893 propertyName + " not found"); 2894 } else { 2895 JsonFactory dumpFactory = new JsonFactory(); 2896 JsonGenerator dumpGenerator = dumpFactory.createJsonGenerator(out); 2897 dumpGenerator.writeStartObject(); 2898 dumpGenerator.writeFieldName("property"); 2899 appendJSONProperty(dumpGenerator, config, propertyName, 2900 new ConfigRedactor(config)); 2901 dumpGenerator.writeEndObject(); 2902 dumpGenerator.flush(); 2903 } 2904 } 2905 2906 /** 2907 * Writes out all properties and their attributes (final and resource) to 2908 * the given {@link Writer}, the format of the output would be, 2909 * 2910 * <pre> 2911 * { "properties" : 2912 * [ { key : "key1", 2913 * value : "value1", 2914 * isFinal : "key1.isFinal", 2915 * resource : "key1.resource" }, 2916 * { key : "key2", 2917 * value : "value2", 2918 * isFinal : "ke2.isFinal", 2919 * resource : "key2.resource" } 2920 * ] 2921 * } 2922 * </pre> 2923 * 2924 * It does not output the properties of the configuration object which 2925 * is loaded from an input stream. 2926 * <p> 2927 * 2928 * @param config the configuration 2929 * @param out the Writer to write to 2930 * @throws IOException 2931 */ 2932 public static void dumpConfiguration(Configuration config, 2933 Writer out) throws IOException { 2934 JsonFactory dumpFactory = new JsonFactory(); 2935 JsonGenerator dumpGenerator = dumpFactory.createJsonGenerator(out); 2936 dumpGenerator.writeStartObject(); 2937 dumpGenerator.writeFieldName("properties"); 2938 dumpGenerator.writeStartArray(); 2939 dumpGenerator.flush(); 2940 ConfigRedactor redactor = new ConfigRedactor(config); 2941 synchronized (config) { 2942 for (Map.Entry<Object,Object> item: config.getProps().entrySet()) { 2943 appendJSONProperty(dumpGenerator, config, item.getKey().toString(), 2944 redactor); 2945 } 2946 } 2947 dumpGenerator.writeEndArray(); 2948 dumpGenerator.writeEndObject(); 2949 dumpGenerator.flush(); 2950 } 2951 2952 /** 2953 * Write property and its attributes as json format to given 2954 * {@link JsonGenerator}. 2955 * 2956 * @param jsonGen json writer 2957 * @param config configuration 2958 * @param name property name 2959 * @throws IOException 2960 */ 2961 private static void appendJSONProperty(JsonGenerator jsonGen, 2962 Configuration config, String name, ConfigRedactor redactor) 2963 throws IOException { 2964 // skip writing if given property name is empty or null 2965 if(!Strings.isNullOrEmpty(name) && jsonGen != null) { 2966 jsonGen.writeStartObject(); 2967 jsonGen.writeStringField("key", name); 2968 jsonGen.writeStringField("value", 2969 redactor.redact(name, config.get(name))); 2970 jsonGen.writeBooleanField("isFinal", 2971 config.finalParameters.contains(name)); 2972 String[] resources = config.updatingResource.get(name); 2973 String resource = UNKNOWN_RESOURCE; 2974 if(resources != null && resources.length > 0) { 2975 resource = resources[0]; 2976 } 2977 jsonGen.writeStringField("resource", resource); 2978 jsonGen.writeEndObject(); 2979 } 2980 } 2981 2982 /** 2983 * Get the {@link ClassLoader} for this job. 2984 * 2985 * @return the correct class loader. 2986 */ 2987 public ClassLoader getClassLoader() { 2988 return classLoader; 2989 } 2990 2991 /** 2992 * Set the class loader that will be used to load the various objects. 2993 * 2994 * @param classLoader the new class loader. 2995 */ 2996 public void setClassLoader(ClassLoader classLoader) { 2997 this.classLoader = classLoader; 2998 } 2999 3000 @Override 3001 public String toString() { 3002 StringBuilder sb = new StringBuilder(); 3003 sb.append("Configuration: "); 3004 if(loadDefaults) { 3005 toString(defaultResources, sb); 3006 if(resources.size()>0) { 3007 sb.append(", "); 3008 } 3009 } 3010 toString(resources, sb); 3011 return sb.toString(); 3012 } 3013 3014 private <T> void toString(List<T> resources, StringBuilder sb) { 3015 ListIterator<T> i = resources.listIterator(); 3016 while (i.hasNext()) { 3017 if (i.nextIndex() != 0) { 3018 sb.append(", "); 3019 } 3020 sb.append(i.next()); 3021 } 3022 } 3023 3024 /** 3025 * Set the quietness-mode. 3026 * 3027 * In the quiet-mode, error and informational messages might not be logged. 3028 * 3029 * @param quietmode <code>true</code> to set quiet-mode on, <code>false</code> 3030 * to turn it off. 3031 */ 3032 public synchronized void setQuietMode(boolean quietmode) { 3033 this.quietmode = quietmode; 3034 } 3035 3036 synchronized boolean getQuietMode() { 3037 return this.quietmode; 3038 } 3039 3040 /** For debugging. List non-default properties to the terminal and exit. */ 3041 public static void main(String[] args) throws Exception { 3042 new Configuration().writeXml(System.out); 3043 } 3044 3045 @Override 3046 public void readFields(DataInput in) throws IOException { 3047 clear(); 3048 int size = WritableUtils.readVInt(in); 3049 for(int i=0; i < size; ++i) { 3050 String key = org.apache.hadoop.io.Text.readString(in); 3051 String value = org.apache.hadoop.io.Text.readString(in); 3052 set(key, value); 3053 String sources[] = WritableUtils.readCompressedStringArray(in); 3054 if(sources != null) { 3055 updatingResource.put(key, sources); 3056 } 3057 } 3058 } 3059 3060 //@Override 3061 @Override 3062 public void write(DataOutput out) throws IOException { 3063 Properties props = getProps(); 3064 WritableUtils.writeVInt(out, props.size()); 3065 for(Map.Entry<Object, Object> item: props.entrySet()) { 3066 org.apache.hadoop.io.Text.writeString(out, (String) item.getKey()); 3067 org.apache.hadoop.io.Text.writeString(out, (String) item.getValue()); 3068 WritableUtils.writeCompressedStringArray(out, 3069 updatingResource.get(item.getKey())); 3070 } 3071 } 3072 3073 /** 3074 * get keys matching the the regex 3075 * @param regex 3076 * @return Map<String,String> with matching keys 3077 */ 3078 public Map<String,String> getValByRegex(String regex) { 3079 Pattern p = Pattern.compile(regex); 3080 3081 Map<String,String> result = new HashMap<String,String>(); 3082 Matcher m; 3083 3084 for(Map.Entry<Object,Object> item: getProps().entrySet()) { 3085 if (item.getKey() instanceof String && 3086 item.getValue() instanceof String) { 3087 m = p.matcher((String)item.getKey()); 3088 if(m.find()) { // match 3089 result.put((String) item.getKey(), 3090 substituteVars(getProps().getProperty((String) item.getKey()))); 3091 } 3092 } 3093 } 3094 return result; 3095 } 3096 3097 /** 3098 * A unique class which is used as a sentinel value in the caching 3099 * for getClassByName. {@see Configuration#getClassByNameOrNull(String)} 3100 */ 3101 private static abstract class NegativeCacheSentinel {} 3102 3103 public static void dumpDeprecatedKeys() { 3104 DeprecationContext deprecations = deprecationContext.get(); 3105 for (Map.Entry<String, DeprecatedKeyInfo> entry : 3106 deprecations.getDeprecatedKeyMap().entrySet()) { 3107 StringBuilder newKeys = new StringBuilder(); 3108 for (String newKey : entry.getValue().newKeys) { 3109 newKeys.append(newKey).append("\t"); 3110 } 3111 System.out.println(entry.getKey() + "\t" + newKeys.toString()); 3112 } 3113 } 3114 3115 /** 3116 * Returns whether or not a deprecated name has been warned. If the name is not 3117 * deprecated then always return false 3118 */ 3119 public static boolean hasWarnedDeprecation(String name) { 3120 DeprecationContext deprecations = deprecationContext.get(); 3121 if(deprecations.getDeprecatedKeyMap().containsKey(name)) { 3122 if(deprecations.getDeprecatedKeyMap().get(name).accessed.get()) { 3123 return true; 3124 } 3125 } 3126 return false; 3127 } 3128}