001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     *
010     *     http://www.apache.org/licenses/LICENSE-2.0
011     *
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    
019    package org.apache.hadoop.conf;
020    
021    import java.io.BufferedInputStream;
022    import java.io.DataInput;
023    import java.io.DataOutput;
024    import java.io.File;
025    import java.io.FileInputStream;
026    import java.io.IOException;
027    import java.io.InputStream;
028    import java.io.InputStreamReader;
029    import java.io.OutputStream;
030    import java.io.OutputStreamWriter;
031    import java.io.Reader;
032    import java.io.Writer;
033    import java.lang.ref.WeakReference;
034    import java.net.InetSocketAddress;
035    import java.net.URL;
036    import java.util.ArrayList;
037    import java.util.Arrays;
038    import java.util.Collection;
039    import java.util.Collections;
040    import java.util.Enumeration;
041    import java.util.HashMap;
042    import java.util.HashSet;
043    import java.util.Iterator;
044    import java.util.LinkedList;
045    import java.util.List;
046    import java.util.ListIterator;
047    import java.util.Map;
048    import java.util.Map.Entry;
049    import java.util.Properties;
050    import java.util.Set;
051    import java.util.StringTokenizer;
052    import java.util.WeakHashMap;
053    import java.util.concurrent.CopyOnWriteArrayList;
054    import java.util.regex.Matcher;
055    import java.util.regex.Pattern;
056    import java.util.regex.PatternSyntaxException;
057    
058    import javax.xml.parsers.DocumentBuilder;
059    import javax.xml.parsers.DocumentBuilderFactory;
060    import javax.xml.parsers.ParserConfigurationException;
061    import javax.xml.transform.Transformer;
062    import javax.xml.transform.TransformerException;
063    import javax.xml.transform.TransformerFactory;
064    import javax.xml.transform.dom.DOMSource;
065    import javax.xml.transform.stream.StreamResult;
066    
067    import org.apache.commons.logging.Log;
068    import org.apache.commons.logging.LogFactory;
069    import org.apache.hadoop.classification.InterfaceAudience;
070    import org.apache.hadoop.classification.InterfaceStability;
071    import org.apache.hadoop.fs.FileSystem;
072    import org.apache.hadoop.fs.Path;
073    import org.apache.hadoop.fs.CommonConfigurationKeys;
074    import org.apache.hadoop.io.Writable;
075    import org.apache.hadoop.io.WritableUtils;
076    import org.apache.hadoop.net.NetUtils;
077    import org.apache.hadoop.util.ReflectionUtils;
078    import org.apache.hadoop.util.StringUtils;
079    import org.codehaus.jackson.JsonFactory;
080    import org.codehaus.jackson.JsonGenerator;
081    import org.w3c.dom.DOMException;
082    import org.w3c.dom.Document;
083    import org.w3c.dom.Element;
084    import org.w3c.dom.Node;
085    import org.w3c.dom.NodeList;
086    import org.w3c.dom.Text;
087    import org.xml.sax.SAXException;
088    
089    /** 
090     * Provides access to configuration parameters.
091     *
092     * <h4 id="Resources">Resources</h4>
093     *
094     * <p>Configurations are specified by resources. A resource contains a set of
095     * name/value pairs as XML data. Each resource is named by either a 
096     * <code>String</code> or by a {@link Path}. If named by a <code>String</code>, 
097     * then the classpath is examined for a file with that name.  If named by a 
098     * <code>Path</code>, then the local filesystem is examined directly, without 
099     * referring to the classpath.
100     *
101     * <p>Unless explicitly turned off, Hadoop by default specifies two 
102     * resources, loaded in-order from the classpath: <ol>
103     * <li><tt><a href="{@docRoot}/../core-default.html">core-default.xml</a>
104     * </tt>: Read-only defaults for hadoop.</li>
105     * <li><tt>core-site.xml</tt>: Site-specific configuration for a given hadoop
106     * installation.</li>
107     * </ol>
108     * Applications may add additional resources, which are loaded
109     * subsequent to these resources in the order they are added.
110     * 
111     * <h4 id="FinalParams">Final Parameters</h4>
112     *
113     * <p>Configuration parameters may be declared <i>final</i>. 
114     * Once a resource declares a value final, no subsequently-loaded 
115     * resource can alter that value.  
116     * For example, one might define a final parameter with:
117     * <tt><pre>
118     *  &lt;property&gt;
119     *    &lt;name&gt;dfs.client.buffer.dir&lt;/name&gt;
120     *    &lt;value&gt;/tmp/hadoop/dfs/client&lt;/value&gt;
121     *    <b>&lt;final&gt;true&lt;/final&gt;</b>
122     *  &lt;/property&gt;</pre></tt>
123     *
124     * Administrators typically define parameters as final in 
125     * <tt>core-site.xml</tt> for values that user applications may not alter.
126     *
127     * <h4 id="VariableExpansion">Variable Expansion</h4>
128     *
129     * <p>Value strings are first processed for <i>variable expansion</i>. The
130     * available properties are:<ol>
131     * <li>Other properties defined in this Configuration; and, if a name is
132     * undefined here,</li>
133     * <li>Properties in {@link System#getProperties()}.</li>
134     * </ol>
135     *
136     * <p>For example, if a configuration resource contains the following property
137     * definitions: 
138     * <tt><pre>
139     *  &lt;property&gt;
140     *    &lt;name&gt;basedir&lt;/name&gt;
141     *    &lt;value&gt;/user/${<i>user.name</i>}&lt;/value&gt;
142     *  &lt;/property&gt;
143     *  
144     *  &lt;property&gt;
145     *    &lt;name&gt;tempdir&lt;/name&gt;
146     *    &lt;value&gt;${<i>basedir</i>}/tmp&lt;/value&gt;
147     *  &lt;/property&gt;</pre></tt>
148     *
149     * When <tt>conf.get("tempdir")</tt> is called, then <tt>${<i>basedir</i>}</tt>
150     * will be resolved to another property in this Configuration, while
151     * <tt>${<i>user.name</i>}</tt> would then ordinarily be resolved to the value
152     * of the System property with that name.
153     */
154    @InterfaceAudience.Public
155    @InterfaceStability.Stable
156    public class Configuration implements Iterable<Map.Entry<String,String>>,
157                                          Writable {
158      private static final Log LOG =
159        LogFactory.getLog(Configuration.class);
160    
161      private boolean quietmode = true;
162      
163      private static class Resource {
164        private final Object resource;
165        private final String name;
166        
167        public Resource(Object resource) {
168          this(resource, resource.toString());
169        }
170        
171        public Resource(Object resource, String name) {
172          this.resource = resource;
173          this.name = name;
174        }
175        
176        public String getName(){
177          return name;
178        }
179        
180        public Object getResource() {
181          return resource;
182        }
183        
184        @Override
185        public String toString() {
186          return name;
187        }
188      }
189      
190      /**
191       * List of configuration resources.
192       */
193      private ArrayList<Resource> resources = new ArrayList<Resource>();
194      
195      /**
196       * The value reported as the setting resource when a key is set
197       * by code rather than a file resource by dumpConfiguration.
198       */
199      static final String UNKNOWN_RESOURCE = "Unknown";
200    
201    
202      /**
203       * List of configuration parameters marked <b>final</b>. 
204       */
205      private Set<String> finalParameters = new HashSet<String>();
206      
207      private boolean loadDefaults = true;
208      
209      /**
210       * Configuration objects
211       */
212      private static final WeakHashMap<Configuration,Object> REGISTRY = 
213        new WeakHashMap<Configuration,Object>();
214      
215      /**
216       * List of default Resources. Resources are loaded in the order of the list 
217       * entries
218       */
219      private static final CopyOnWriteArrayList<String> defaultResources =
220        new CopyOnWriteArrayList<String>();
221    
222      private static final Map<ClassLoader, Map<String, WeakReference<Class<?>>>>
223        CACHE_CLASSES = new WeakHashMap<ClassLoader, Map<String, WeakReference<Class<?>>>>();
224    
225      /**
226       * Sentinel value to store negative cache results in {@link #CACHE_CLASSES}.
227       */
228      private static final Class<?> NEGATIVE_CACHE_SENTINEL =
229        NegativeCacheSentinel.class;
230    
231      /**
232       * Stores the mapping of key to the resource which modifies or loads 
233       * the key most recently
234       */
235      private HashMap<String, String[]> updatingResource;
236     
237      /**
238       * Class to keep the information about the keys which replace the deprecated
239       * ones.
240       * 
241       * This class stores the new keys which replace the deprecated keys and also
242       * gives a provision to have a custom message for each of the deprecated key
243       * that is being replaced. It also provides method to get the appropriate
244       * warning message which can be logged whenever the deprecated key is used.
245       */
246      private static class DeprecatedKeyInfo {
247        private String[] newKeys;
248        private String customMessage;
249        private boolean accessed;
250        DeprecatedKeyInfo(String[] newKeys, String customMessage) {
251          this.newKeys = newKeys;
252          this.customMessage = customMessage;
253          accessed = false;
254        }
255    
256        /**
257         * Method to provide the warning message. It gives the custom message if
258         * non-null, and default message otherwise.
259         * @param key the associated deprecated key.
260         * @return message that is to be logged when a deprecated key is used.
261         */
262        private final String getWarningMessage(String key) {
263          String warningMessage;
264          if(customMessage == null) {
265            StringBuilder message = new StringBuilder(key);
266            String deprecatedKeySuffix = " is deprecated. Instead, use ";
267            message.append(deprecatedKeySuffix);
268            for (int i = 0; i < newKeys.length; i++) {
269              message.append(newKeys[i]);
270              if(i != newKeys.length-1) {
271                message.append(", ");
272              }
273            }
274            warningMessage = message.toString();
275          }
276          else {
277            warningMessage = customMessage;
278          }
279          accessed = true;
280          return warningMessage;
281        }
282      }
283      
284      /**
285       * Stores the deprecated keys, the new keys which replace the deprecated keys
286       * and custom message(if any provided).
287       */
288      private static Map<String, DeprecatedKeyInfo> deprecatedKeyMap = 
289          new HashMap<String, DeprecatedKeyInfo>();
290      
291      /**
292       * Stores a mapping from superseding keys to the keys which they deprecate.
293       */
294      private static Map<String, String> reverseDeprecatedKeyMap =
295          new HashMap<String, String>();
296    
297      /**
298       * Adds the deprecated key to the deprecation map.
299       * It does not override any existing entries in the deprecation map.
300       * This is to be used only by the developers in order to add deprecation of
301       * keys, and attempts to call this method after loading resources once,
302       * would lead to <tt>UnsupportedOperationException</tt>
303       * 
304       * If a key is deprecated in favor of multiple keys, they are all treated as 
305       * aliases of each other, and setting any one of them resets all the others 
306       * to the new value.
307       * 
308       * @param key
309       * @param newKeys
310       * @param customMessage
311       * @deprecated use {@link #addDeprecation(String key, String newKey,
312          String customMessage)} instead
313       */
314      @Deprecated
315      public synchronized static void addDeprecation(String key, String[] newKeys,
316          String customMessage) {
317        if (key == null || key.length() == 0 ||
318            newKeys == null || newKeys.length == 0) {
319          throw new IllegalArgumentException();
320        }
321        if (!isDeprecated(key)) {
322          DeprecatedKeyInfo newKeyInfo;
323          newKeyInfo = new DeprecatedKeyInfo(newKeys, customMessage);
324          deprecatedKeyMap.put(key, newKeyInfo);
325          for (String newKey : newKeys) {
326            reverseDeprecatedKeyMap.put(newKey, key);
327          }
328        }
329      }
330      
331      /**
332       * Adds the deprecated key to the deprecation map.
333       * It does not override any existing entries in the deprecation map.
334       * This is to be used only by the developers in order to add deprecation of
335       * keys, and attempts to call this method after loading resources once,
336       * would lead to <tt>UnsupportedOperationException</tt>
337       * 
338       * @param key
339       * @param newKey
340       * @param customMessage
341       */
342      public synchronized static void addDeprecation(String key, String newKey,
343                  String customMessage) {
344              addDeprecation(key, new String[] {newKey}, customMessage);
345      }
346    
347      /**
348       * Adds the deprecated key to the deprecation map when no custom message
349       * is provided.
350       * It does not override any existing entries in the deprecation map.
351       * This is to be used only by the developers in order to add deprecation of
352       * keys, and attempts to call this method after loading resources once,
353       * would lead to <tt>UnsupportedOperationException</tt>
354       * 
355       * If a key is deprecated in favor of multiple keys, they are all treated as 
356       * aliases of each other, and setting any one of them resets all the others 
357       * to the new value.
358       * 
359       * @param key Key that is to be deprecated
360       * @param newKeys list of keys that take up the values of deprecated key
361       * @deprecated use {@link #addDeprecation(String key, String newKey)} instead
362       */
363      @Deprecated
364      public synchronized static void addDeprecation(String key, String[] newKeys) {
365        addDeprecation(key, newKeys, null);
366      }
367      
368      /**
369       * Adds the deprecated key to the deprecation map when no custom message
370       * is provided.
371       * It does not override any existing entries in the deprecation map.
372       * This is to be used only by the developers in order to add deprecation of
373       * keys, and attempts to call this method after loading resources once,
374       * would lead to <tt>UnsupportedOperationException</tt>
375       * 
376       * @param key Key that is to be deprecated
377       * @param newKey key that takes up the value of deprecated key
378       */
379      public synchronized static void addDeprecation(String key, String newKey) {
380            addDeprecation(key, new String[] {newKey}, null);
381      }
382      
383      /**
384       * checks whether the given <code>key</code> is deprecated.
385       * 
386       * @param key the parameter which is to be checked for deprecation
387       * @return <code>true</code> if the key is deprecated and 
388       *         <code>false</code> otherwise.
389       */
390      public static boolean isDeprecated(String key) {
391        return deprecatedKeyMap.containsKey(key);
392      }
393    
394      /**
395       * Returns the alternate name for a key if the property name is deprecated
396       * or if deprecates a property name.
397       *
398       * @param name property name.
399       * @return alternate name.
400       */
401      private String[] getAlternateNames(String name) {
402        String altNames[] = null;
403        DeprecatedKeyInfo keyInfo = deprecatedKeyMap.get(name);
404        if (keyInfo == null) {
405          altNames = (reverseDeprecatedKeyMap.get(name) != null ) ? 
406            new String [] {reverseDeprecatedKeyMap.get(name)} : null;
407          if(altNames != null && altNames.length > 0) {
408            //To help look for other new configs for this deprecated config
409            keyInfo = deprecatedKeyMap.get(altNames[0]);
410          }      
411        } 
412        if(keyInfo != null && keyInfo.newKeys.length > 0) {
413          List<String> list = new ArrayList<String>(); 
414          if(altNames != null) {
415              list.addAll(Arrays.asList(altNames));
416          }
417          list.addAll(Arrays.asList(keyInfo.newKeys));
418          altNames = list.toArray(new String[list.size()]);
419        }
420        return altNames;
421      }
422    
423      /**
424       * Checks for the presence of the property <code>name</code> in the
425       * deprecation map. Returns the first of the list of new keys if present
426       * in the deprecation map or the <code>name</code> itself. If the property
427       * is not presently set but the property map contains an entry for the
428       * deprecated key, the value of the deprecated key is set as the value for
429       * the provided property name.
430       *
431       * @param name the property name
432       * @return the first property in the list of properties mapping
433       *         the <code>name</code> or the <code>name</code> itself.
434       */
435      private String[] handleDeprecation(String name) {
436        ArrayList<String > names = new ArrayList<String>();
437            if (isDeprecated(name)) {
438          DeprecatedKeyInfo keyInfo = deprecatedKeyMap.get(name);
439          warnOnceIfDeprecated(name);
440          for (String newKey : keyInfo.newKeys) {
441            if(newKey != null) {
442              names.add(newKey);
443            }
444          }
445        }
446        if(names.size() == 0) {
447            names.add(name);
448        }
449        for(String n : names) {
450              String deprecatedKey = reverseDeprecatedKeyMap.get(n);
451              if (deprecatedKey != null && !getOverlay().containsKey(n) &&
452                  getOverlay().containsKey(deprecatedKey)) {
453                getProps().setProperty(n, getOverlay().getProperty(deprecatedKey));
454                getOverlay().setProperty(n, getOverlay().getProperty(deprecatedKey));
455              }
456        }
457        return names.toArray(new String[names.size()]);
458      }
459     
460      private void handleDeprecation() {
461        LOG.debug("Handling deprecation for all properties in config...");
462        Set<Object> keys = new HashSet<Object>();
463        keys.addAll(getProps().keySet());
464        for (Object item: keys) {
465          LOG.debug("Handling deprecation for " + (String)item);
466          handleDeprecation((String)item);
467        }
468      }
469     
470      static{
471        //print deprecation warning if hadoop-site.xml is found in classpath
472        ClassLoader cL = Thread.currentThread().getContextClassLoader();
473        if (cL == null) {
474          cL = Configuration.class.getClassLoader();
475        }
476        if(cL.getResource("hadoop-site.xml")!=null) {
477          LOG.warn("DEPRECATED: hadoop-site.xml found in the classpath. " +
478              "Usage of hadoop-site.xml is deprecated. Instead use core-site.xml, "
479              + "mapred-site.xml and hdfs-site.xml to override properties of " +
480              "core-default.xml, mapred-default.xml and hdfs-default.xml " +
481              "respectively");
482        }
483        addDefaultResource("core-default.xml");
484        addDefaultResource("core-site.xml");
485        //Add code for managing deprecated key mapping
486        //for example
487        //addDeprecation("oldKey1",new String[]{"newkey1","newkey2"});
488        //adds deprecation for oldKey1 to two new keys(newkey1, newkey2).
489        //so get or set of oldKey1 will correctly populate/access values of 
490        //newkey1 and newkey2
491        addDeprecatedKeys();
492      }
493      
494      private Properties properties;
495      private Properties overlay;
496      private ClassLoader classLoader;
497      {
498        classLoader = Thread.currentThread().getContextClassLoader();
499        if (classLoader == null) {
500          classLoader = Configuration.class.getClassLoader();
501        }
502      }
503      
504      /** A new configuration. */
505      public Configuration() {
506        this(true);
507      }
508    
509      /** A new configuration where the behavior of reading from the default 
510       * resources can be turned off.
511       * 
512       * If the parameter {@code loadDefaults} is false, the new instance
513       * will not load resources from the default files. 
514       * @param loadDefaults specifies whether to load from the default files
515       */
516      public Configuration(boolean loadDefaults) {
517        this.loadDefaults = loadDefaults;
518        updatingResource = new HashMap<String, String[]>();
519        synchronized(Configuration.class) {
520          REGISTRY.put(this, null);
521        }
522      }
523      
524      /** 
525       * A new configuration with the same settings cloned from another.
526       * 
527       * @param other the configuration from which to clone settings.
528       */
529      @SuppressWarnings("unchecked")
530      public Configuration(Configuration other) {
531       this.resources = (ArrayList<Resource>) other.resources.clone();
532       synchronized(other) {
533         if (other.properties != null) {
534           this.properties = (Properties)other.properties.clone();
535         }
536    
537         if (other.overlay!=null) {
538           this.overlay = (Properties)other.overlay.clone();
539         }
540    
541         this.updatingResource = new HashMap<String, String[]>(other.updatingResource);
542       }
543       
544        this.finalParameters = new HashSet<String>(other.finalParameters);
545        synchronized(Configuration.class) {
546          REGISTRY.put(this, null);
547        }
548        this.classLoader = other.classLoader;
549        this.loadDefaults = other.loadDefaults;
550        setQuietMode(other.getQuietMode());
551      }
552      
553      /**
554       * Add a default resource. Resources are loaded in the order of the resources 
555       * added.
556       * @param name file name. File should be present in the classpath.
557       */
558      public static synchronized void addDefaultResource(String name) {
559        if(!defaultResources.contains(name)) {
560          defaultResources.add(name);
561          for(Configuration conf : REGISTRY.keySet()) {
562            if(conf.loadDefaults) {
563              conf.reloadConfiguration();
564            }
565          }
566        }
567      }
568    
569      /**
570       * Add a configuration resource. 
571       * 
572       * The properties of this resource will override properties of previously 
573       * added resources, unless they were marked <a href="#Final">final</a>. 
574       * 
575       * @param name resource to be added, the classpath is examined for a file 
576       *             with that name.
577       */
578      public void addResource(String name) {
579        addResourceObject(new Resource(name));
580      }
581    
582      /**
583       * Add a configuration resource. 
584       * 
585       * The properties of this resource will override properties of previously 
586       * added resources, unless they were marked <a href="#Final">final</a>. 
587       * 
588       * @param url url of the resource to be added, the local filesystem is 
589       *            examined directly to find the resource, without referring to 
590       *            the classpath.
591       */
592      public void addResource(URL url) {
593        addResourceObject(new Resource(url));
594      }
595    
596      /**
597       * Add a configuration resource. 
598       * 
599       * The properties of this resource will override properties of previously 
600       * added resources, unless they were marked <a href="#Final">final</a>. 
601       * 
602       * @param file file-path of resource to be added, the local filesystem is
603       *             examined directly to find the resource, without referring to 
604       *             the classpath.
605       */
606      public void addResource(Path file) {
607        addResourceObject(new Resource(file));
608      }
609    
610      /**
611       * Add a configuration resource. 
612       * 
613       * The properties of this resource will override properties of previously 
614       * added resources, unless they were marked <a href="#Final">final</a>. 
615       * 
616       * WARNING: The contents of the InputStream will be cached, by this method. 
617       * So use this sparingly because it does increase the memory consumption.
618       * 
619       * @param in InputStream to deserialize the object from. In will be read from
620       * when a get or set is called next.  After it is read the stream will be
621       * closed. 
622       */
623      public void addResource(InputStream in) {
624        addResourceObject(new Resource(in));
625      }
626    
627      /**
628       * Add a configuration resource. 
629       * 
630       * The properties of this resource will override properties of previously 
631       * added resources, unless they were marked <a href="#Final">final</a>. 
632       * 
633       * @param in InputStream to deserialize the object from.
634       * @param name the name of the resource because InputStream.toString is not
635       * very descriptive some times.  
636       */
637      public void addResource(InputStream in, String name) {
638        addResourceObject(new Resource(in, name));
639      }
640      
641      
642      /**
643       * Reload configuration from previously added resources.
644       *
645       * This method will clear all the configuration read from the added 
646       * resources, and final parameters. This will make the resources to 
647       * be read again before accessing the values. Values that are added
648       * via set methods will overlay values read from the resources.
649       */
650      public synchronized void reloadConfiguration() {
651        properties = null;                            // trigger reload
652        finalParameters.clear();                      // clear site-limits
653      }
654      
655      private synchronized void addResourceObject(Resource resource) {
656        resources.add(resource);                      // add to resources
657        reloadConfiguration();
658      }
659      
660      private static Pattern varPat = Pattern.compile("\\$\\{[^\\}\\$\u0020]+\\}");
661      private static int MAX_SUBST = 20;
662    
663      private String substituteVars(String expr) {
664        if (expr == null) {
665          return null;
666        }
667        Matcher match = varPat.matcher("");
668        String eval = expr;
669        for(int s=0; s<MAX_SUBST; s++) {
670          match.reset(eval);
671          if (!match.find()) {
672            return eval;
673          }
674          String var = match.group();
675          var = var.substring(2, var.length()-1); // remove ${ .. }
676          String val = null;
677          try {
678            val = System.getProperty(var);
679          } catch(SecurityException se) {
680            LOG.warn("Unexpected SecurityException in Configuration", se);
681          }
682          if (val == null) {
683            val = getRaw(var);
684          }
685          if (val == null) {
686            return eval; // return literal ${var}: var is unbound
687          }
688          // substitute
689          eval = eval.substring(0, match.start())+val+eval.substring(match.end());
690        }
691        throw new IllegalStateException("Variable substitution depth too large: " 
692                                        + MAX_SUBST + " " + expr);
693      }
694      
695      /**
696       * Get the value of the <code>name</code> property, <code>null</code> if
697       * no such property exists. If the key is deprecated, it returns the value of
698       * the first key which replaces the deprecated key and is not null
699       * 
700       * Values are processed for <a href="#VariableExpansion">variable expansion</a> 
701       * before being returned. 
702       * 
703       * @param name the property name.
704       * @return the value of the <code>name</code> or its replacing property, 
705       *         or null if no such property exists.
706       */
707      public String get(String name) {
708        String[] names = handleDeprecation(name);
709        String result = null;
710        for(String n : names) {
711          result = substituteVars(getProps().getProperty(n));
712        }
713        return result;
714      }
715      
716      /**
717       * Get the value of the <code>name</code> property as a trimmed <code>String</code>, 
718       * <code>null</code> if no such property exists. 
719       * If the key is deprecated, it returns the value of
720       * the first key which replaces the deprecated key and is not null
721       * 
722       * Values are processed for <a href="#VariableExpansion">variable expansion</a> 
723       * before being returned. 
724       * 
725       * @param name the property name.
726       * @return the value of the <code>name</code> or its replacing property, 
727       *         or null if no such property exists.
728       */
729      public String getTrimmed(String name) {
730        String value = get(name);
731        
732        if (null == value) {
733          return null;
734        } else {
735          return value.trim();
736        }
737      }
738    
739      /**
740       * Get the value of the <code>name</code> property, without doing
741       * <a href="#VariableExpansion">variable expansion</a>.If the key is 
742       * deprecated, it returns the value of the first key which replaces 
743       * the deprecated key and is not null.
744       * 
745       * @param name the property name.
746       * @return the value of the <code>name</code> property or 
747       *         its replacing property and null if no such property exists.
748       */
749      public String getRaw(String name) {
750        String[] names = handleDeprecation(name);
751        String result = null;
752        for(String n : names) {
753          result = getProps().getProperty(n);
754        }
755        return result;
756      }
757    
758      /** 
759       * Set the <code>value</code> of the <code>name</code> property. If 
760       * <code>name</code> is deprecated or there is a deprecated name associated to it,
761       * it sets the value to both names.
762       * 
763       * @param name property name.
764       * @param value property value.
765       */
766      public void set(String name, String value) {
767        set(name, value, null);
768      }
769      
770      /** 
771       * Set the <code>value</code> of the <code>name</code> property. If 
772       * <code>name</code> is deprecated or there is a deprecated name associated to it,
773       * it sets the value to both names.
774       * 
775       * @param name property name.
776       * @param value property value.
777       * @param source the place that this configuration value came from 
778       * (For debugging).
779       */
780      public void set(String name, String value, String source) {
781        if (deprecatedKeyMap.isEmpty()) {
782          getProps();
783        }
784        getOverlay().setProperty(name, value);
785        getProps().setProperty(name, value);
786        if(source == null) {
787          updatingResource.put(name, new String[] {"programatically"});
788        } else {
789          updatingResource.put(name, new String[] {source});
790        }
791        String[] altNames = getAlternateNames(name);
792        if (altNames != null && altNames.length > 0) {
793          String altSource = "because " + name + " is deprecated";
794          for(String altName : altNames) {
795            if(!altName.equals(name)) {
796              getOverlay().setProperty(altName, value);
797              getProps().setProperty(altName, value);
798              updatingResource.put(altName, new String[] {altSource});
799            }
800          }
801        }
802        warnOnceIfDeprecated(name);
803      }
804    
805      private void warnOnceIfDeprecated(String name) {
806        DeprecatedKeyInfo keyInfo = deprecatedKeyMap.get(name);
807        if (keyInfo != null && !keyInfo.accessed) {
808          LOG.warn(keyInfo.getWarningMessage(name));
809        }
810      }
811    
812      /**
813       * Unset a previously set property.
814       */
815      public synchronized void unset(String name) {
816        String[] altNames = getAlternateNames(name);
817        getOverlay().remove(name);
818        getProps().remove(name);
819        if (altNames !=null && altNames.length > 0) {
820          for(String altName : altNames) {
821            getOverlay().remove(altName);
822            getProps().remove(altName);
823          }
824        }
825      }
826    
827      /**
828       * Sets a property if it is currently unset.
829       * @param name the property name
830       * @param value the new value
831       */
832      public synchronized void setIfUnset(String name, String value) {
833        if (get(name) == null) {
834          set(name, value);
835        }
836      }
837      
838      private synchronized Properties getOverlay() {
839        if (overlay==null){
840          overlay=new Properties();
841        }
842        return overlay;
843      }
844    
845      /** 
846       * Get the value of the <code>name</code>. If the key is deprecated,
847       * it returns the value of the first key which replaces the deprecated key
848       * and is not null.
849       * If no such property exists,
850       * then <code>defaultValue</code> is returned.
851       * 
852       * @param name property name.
853       * @param defaultValue default value.
854       * @return property value, or <code>defaultValue</code> if the property 
855       *         doesn't exist.                    
856       */
857      public String get(String name, String defaultValue) {
858        String[] names = handleDeprecation(name);
859        String result = null;
860        for(String n : names) {
861          result = substituteVars(getProps().getProperty(n, defaultValue));
862        }
863        return result;
864      }
865        
866      /** 
867       * Get the value of the <code>name</code> property as an <code>int</code>.
868       *   
869       * If no such property exists, the provided default value is returned,
870       * or if the specified value is not a valid <code>int</code>,
871       * then an error is thrown.
872       * 
873       * @param name property name.
874       * @param defaultValue default value.
875       * @throws NumberFormatException when the value is invalid
876       * @return property value as an <code>int</code>, 
877       *         or <code>defaultValue</code>. 
878       */
879      public int getInt(String name, int defaultValue) {
880        String valueString = getTrimmed(name);
881        if (valueString == null)
882          return defaultValue;
883        String hexString = getHexDigits(valueString);
884        if (hexString != null) {
885          return Integer.parseInt(hexString, 16);
886        }
887        return Integer.parseInt(valueString);
888      }
889      
890      /**
891       * Get the value of the <code>name</code> property as a set of comma-delimited
892       * <code>int</code> values.
893       * 
894       * If no such property exists, an empty array is returned.
895       * 
896       * @param name property name
897       * @return property value interpreted as an array of comma-delimited
898       *         <code>int</code> values
899       */
900      public int[] getInts(String name) {
901        String[] strings = getTrimmedStrings(name);
902        int[] ints = new int[strings.length];
903        for (int i = 0; i < strings.length; i++) {
904          ints[i] = Integer.parseInt(strings[i]);
905        }
906        return ints;
907      }
908    
909      /** 
910       * Set the value of the <code>name</code> property to an <code>int</code>.
911       * 
912       * @param name property name.
913       * @param value <code>int</code> value of the property.
914       */
915      public void setInt(String name, int value) {
916        set(name, Integer.toString(value));
917      }
918    
919    
920      /** 
921       * Get the value of the <code>name</code> property as a <code>long</code>.  
922       * If no such property exists, the provided default value is returned,
923       * or if the specified value is not a valid <code>long</code>,
924       * then an error is thrown.
925       * 
926       * @param name property name.
927       * @param defaultValue default value.
928       * @throws NumberFormatException when the value is invalid
929       * @return property value as a <code>long</code>, 
930       *         or <code>defaultValue</code>. 
931       */
932      public long getLong(String name, long defaultValue) {
933        String valueString = getTrimmed(name);
934        if (valueString == null)
935          return defaultValue;
936        String hexString = getHexDigits(valueString);
937        if (hexString != null) {
938          return Long.parseLong(hexString, 16);
939        }
940        return Long.parseLong(valueString);
941      }
942    
943      /**
944       * Get the value of the <code>name</code> property as a <code>long</code> or
945       * human readable format. If no such property exists, the provided default
946       * value is returned, or if the specified value is not a valid
947       * <code>long</code> or human readable format, then an error is thrown. You
948       * can use the following suffix (case insensitive): k(kilo), m(mega), g(giga),
949       * t(tera), p(peta), e(exa)
950       *
951       * @param name property name.
952       * @param defaultValue default value.
953       * @throws NumberFormatException when the value is invalid
954       * @return property value as a <code>long</code>,
955       *         or <code>defaultValue</code>.
956       */
957      public long getLongBytes(String name, long defaultValue) {
958        String valueString = getTrimmed(name);
959        if (valueString == null)
960          return defaultValue;
961        return StringUtils.TraditionalBinaryPrefix.string2long(valueString);
962      }
963    
964      private String getHexDigits(String value) {
965        boolean negative = false;
966        String str = value;
967        String hexString = null;
968        if (value.startsWith("-")) {
969          negative = true;
970          str = value.substring(1);
971        }
972        if (str.startsWith("0x") || str.startsWith("0X")) {
973          hexString = str.substring(2);
974          if (negative) {
975            hexString = "-" + hexString;
976          }
977          return hexString;
978        }
979        return null;
980      }
981      
982      /** 
983       * Set the value of the <code>name</code> property to a <code>long</code>.
984       * 
985       * @param name property name.
986       * @param value <code>long</code> value of the property.
987       */
988      public void setLong(String name, long value) {
989        set(name, Long.toString(value));
990      }
991    
992      /** 
993       * Get the value of the <code>name</code> property as a <code>float</code>.  
994       * If no such property exists, the provided default value is returned,
995       * or if the specified value is not a valid <code>float</code>,
996       * then an error is thrown.
997       *
998       * @param name property name.
999       * @param defaultValue default value.
1000       * @throws NumberFormatException when the value is invalid
1001       * @return property value as a <code>float</code>, 
1002       *         or <code>defaultValue</code>. 
1003       */
1004      public float getFloat(String name, float defaultValue) {
1005        String valueString = getTrimmed(name);
1006        if (valueString == null)
1007          return defaultValue;
1008        return Float.parseFloat(valueString);
1009      }
1010      /**
1011       * Set the value of the <code>name</code> property to a <code>float</code>.
1012       * 
1013       * @param name property name.
1014       * @param value property value.
1015       */
1016      public void setFloat(String name, float value) {
1017        set(name,Float.toString(value));
1018      }
1019     
1020      /** 
1021       * Get the value of the <code>name</code> property as a <code>boolean</code>.  
1022       * If no such property is specified, or if the specified value is not a valid
1023       * <code>boolean</code>, then <code>defaultValue</code> is returned.
1024       * 
1025       * @param name property name.
1026       * @param defaultValue default value.
1027       * @return property value as a <code>boolean</code>, 
1028       *         or <code>defaultValue</code>. 
1029       */
1030      public boolean getBoolean(String name, boolean defaultValue) {
1031        String valueString = getTrimmed(name);
1032        if (null == valueString || "".equals(valueString)) {
1033          return defaultValue;
1034        }
1035    
1036        valueString = valueString.toLowerCase();
1037    
1038        if ("true".equals(valueString))
1039          return true;
1040        else if ("false".equals(valueString))
1041          return false;
1042        else return defaultValue;
1043      }
1044    
1045      /** 
1046       * Set the value of the <code>name</code> property to a <code>boolean</code>.
1047       * 
1048       * @param name property name.
1049       * @param value <code>boolean</code> value of the property.
1050       */
1051      public void setBoolean(String name, boolean value) {
1052        set(name, Boolean.toString(value));
1053      }
1054    
1055      /**
1056       * Set the given property, if it is currently unset.
1057       * @param name property name
1058       * @param value new value
1059       */
1060      public void setBooleanIfUnset(String name, boolean value) {
1061        setIfUnset(name, Boolean.toString(value));
1062      }
1063    
1064      /**
1065       * Set the value of the <code>name</code> property to the given type. This
1066       * is equivalent to <code>set(&lt;name&gt;, value.toString())</code>.
1067       * @param name property name
1068       * @param value new value
1069       */
1070      public <T extends Enum<T>> void setEnum(String name, T value) {
1071        set(name, value.toString());
1072      }
1073    
1074      /**
1075       * Return value matching this enumerated type.
1076       * @param name Property name
1077       * @param defaultValue Value returned if no mapping exists
1078       * @throws IllegalArgumentException If mapping is illegal for the type
1079       * provided
1080       */
1081      public <T extends Enum<T>> T getEnum(String name, T defaultValue) {
1082        final String val = get(name);
1083        return null == val
1084          ? defaultValue
1085          : Enum.valueOf(defaultValue.getDeclaringClass(), val);
1086      }
1087    
1088      /**
1089       * Get the value of the <code>name</code> property as a <code>Pattern</code>.
1090       * If no such property is specified, or if the specified value is not a valid
1091       * <code>Pattern</code>, then <code>DefaultValue</code> is returned.
1092       *
1093       * @param name property name
1094       * @param defaultValue default value
1095       * @return property value as a compiled Pattern, or defaultValue
1096       */
1097      public Pattern getPattern(String name, Pattern defaultValue) {
1098        String valString = get(name);
1099        if (null == valString || "".equals(valString)) {
1100          return defaultValue;
1101        }
1102        try {
1103          return Pattern.compile(valString);
1104        } catch (PatternSyntaxException pse) {
1105          LOG.warn("Regular expression '" + valString + "' for property '" +
1106                   name + "' not valid. Using default", pse);
1107          return defaultValue;
1108        }
1109      }
1110    
1111      /**
1112       * Set the given property to <code>Pattern</code>.
1113       * If the pattern is passed as null, sets the empty pattern which results in
1114       * further calls to getPattern(...) returning the default value.
1115       *
1116       * @param name property name
1117       * @param pattern new value
1118       */
1119      public void setPattern(String name, Pattern pattern) {
1120        if (null == pattern) {
1121          set(name, null);
1122        } else {
1123          set(name, pattern.pattern());
1124        }
1125      }
1126    
1127      /**
1128       * Gets information about why a property was set.  Typically this is the 
1129       * path to the resource objects (file, URL, etc.) the property came from, but
1130       * it can also indicate that it was set programatically, or because of the
1131       * command line.
1132       *
1133       * @param name - The property name to get the source of.
1134       * @return null - If the property or its source wasn't found. Otherwise, 
1135       * returns a list of the sources of the resource.  The older sources are
1136       * the first ones in the list.  So for example if a configuration is set from
1137       * the command line, and then written out to a file that is read back in the
1138       * first entry would indicate that it was set from the command line, while
1139       * the second one would indicate the file that the new configuration was read
1140       * in from.
1141       */
1142      @InterfaceStability.Unstable
1143      public synchronized String[] getPropertySources(String name) {
1144        if (properties == null) {
1145          // If properties is null, it means a resource was newly added
1146          // but the props were cleared so as to load it upon future
1147          // requests. So lets force a load by asking a properties list.
1148          getProps();
1149        }
1150        // Return a null right away if our properties still
1151        // haven't loaded or the resource mapping isn't defined
1152        if (properties == null || updatingResource == null) {
1153          return null;
1154        } else {
1155          String[] source = updatingResource.get(name);
1156          if(source == null) {
1157            return null;
1158          } else {
1159            return Arrays.copyOf(source, source.length);
1160          }
1161        }
1162      }
1163    
1164      /**
1165       * A class that represents a set of positive integer ranges. It parses 
1166       * strings of the form: "2-3,5,7-" where ranges are separated by comma and 
1167       * the lower/upper bounds are separated by dash. Either the lower or upper 
1168       * bound may be omitted meaning all values up to or over. So the string 
1169       * above means 2, 3, 5, and 7, 8, 9, ...
1170       */
1171      public static class IntegerRanges implements Iterable<Integer>{
1172        private static class Range {
1173          int start;
1174          int end;
1175        }
1176        
1177        private static class RangeNumberIterator implements Iterator<Integer> {
1178          Iterator<Range> internal;
1179          int at;
1180          int end;
1181    
1182          public RangeNumberIterator(List<Range> ranges) {
1183            if (ranges != null) {
1184              internal = ranges.iterator();
1185            }
1186            at = -1;
1187            end = -2;
1188          }
1189          
1190          @Override
1191          public boolean hasNext() {
1192            if (at <= end) {
1193              return true;
1194            } else if (internal != null){
1195              return internal.hasNext();
1196            }
1197            return false;
1198          }
1199    
1200          @Override
1201          public Integer next() {
1202            if (at <= end) {
1203              at++;
1204              return at - 1;
1205            } else if (internal != null){
1206              Range found = internal.next();
1207              if (found != null) {
1208                at = found.start;
1209                end = found.end;
1210                at++;
1211                return at - 1;
1212              }
1213            }
1214            return null;
1215          }
1216    
1217          @Override
1218          public void remove() {
1219            throw new UnsupportedOperationException();
1220          }
1221        };
1222    
1223        List<Range> ranges = new ArrayList<Range>();
1224        
1225        public IntegerRanges() {
1226        }
1227        
1228        public IntegerRanges(String newValue) {
1229          StringTokenizer itr = new StringTokenizer(newValue, ",");
1230          while (itr.hasMoreTokens()) {
1231            String rng = itr.nextToken().trim();
1232            String[] parts = rng.split("-", 3);
1233            if (parts.length < 1 || parts.length > 2) {
1234              throw new IllegalArgumentException("integer range badly formed: " + 
1235                                                 rng);
1236            }
1237            Range r = new Range();
1238            r.start = convertToInt(parts[0], 0);
1239            if (parts.length == 2) {
1240              r.end = convertToInt(parts[1], Integer.MAX_VALUE);
1241            } else {
1242              r.end = r.start;
1243            }
1244            if (r.start > r.end) {
1245              throw new IllegalArgumentException("IntegerRange from " + r.start + 
1246                                                 " to " + r.end + " is invalid");
1247            }
1248            ranges.add(r);
1249          }
1250        }
1251    
1252        /**
1253         * Convert a string to an int treating empty strings as the default value.
1254         * @param value the string value
1255         * @param defaultValue the value for if the string is empty
1256         * @return the desired integer
1257         */
1258        private static int convertToInt(String value, int defaultValue) {
1259          String trim = value.trim();
1260          if (trim.length() == 0) {
1261            return defaultValue;
1262          }
1263          return Integer.parseInt(trim);
1264        }
1265    
1266        /**
1267         * Is the given value in the set of ranges
1268         * @param value the value to check
1269         * @return is the value in the ranges?
1270         */
1271        public boolean isIncluded(int value) {
1272          for(Range r: ranges) {
1273            if (r.start <= value && value <= r.end) {
1274              return true;
1275            }
1276          }
1277          return false;
1278        }
1279        
1280        /**
1281         * @return true if there are no values in this range, else false.
1282         */
1283        public boolean isEmpty() {
1284          return ranges == null || ranges.isEmpty();
1285        }
1286        
1287        @Override
1288        public String toString() {
1289          StringBuilder result = new StringBuilder();
1290          boolean first = true;
1291          for(Range r: ranges) {
1292            if (first) {
1293              first = false;
1294            } else {
1295              result.append(',');
1296            }
1297            result.append(r.start);
1298            result.append('-');
1299            result.append(r.end);
1300          }
1301          return result.toString();
1302        }
1303    
1304        @Override
1305        public Iterator<Integer> iterator() {
1306          return new RangeNumberIterator(ranges);
1307        }
1308        
1309      }
1310    
1311      /**
1312       * Parse the given attribute as a set of integer ranges
1313       * @param name the attribute name
1314       * @param defaultValue the default value if it is not set
1315       * @return a new set of ranges from the configured value
1316       */
1317      public IntegerRanges getRange(String name, String defaultValue) {
1318        return new IntegerRanges(get(name, defaultValue));
1319      }
1320    
1321      /** 
1322       * Get the comma delimited values of the <code>name</code> property as 
1323       * a collection of <code>String</code>s.  
1324       * If no such property is specified then empty collection is returned.
1325       * <p>
1326       * This is an optimized version of {@link #getStrings(String)}
1327       * 
1328       * @param name property name.
1329       * @return property value as a collection of <code>String</code>s. 
1330       */
1331      public Collection<String> getStringCollection(String name) {
1332        String valueString = get(name);
1333        return StringUtils.getStringCollection(valueString);
1334      }
1335    
1336      /** 
1337       * Get the comma delimited values of the <code>name</code> property as 
1338       * an array of <code>String</code>s.  
1339       * If no such property is specified then <code>null</code> is returned.
1340       * 
1341       * @param name property name.
1342       * @return property value as an array of <code>String</code>s, 
1343       *         or <code>null</code>. 
1344       */
1345      public String[] getStrings(String name) {
1346        String valueString = get(name);
1347        return StringUtils.getStrings(valueString);
1348      }
1349    
1350      /** 
1351       * Get the comma delimited values of the <code>name</code> property as 
1352       * an array of <code>String</code>s.  
1353       * If no such property is specified then default value is returned.
1354       * 
1355       * @param name property name.
1356       * @param defaultValue The default value
1357       * @return property value as an array of <code>String</code>s, 
1358       *         or default value. 
1359       */
1360      public String[] getStrings(String name, String... defaultValue) {
1361        String valueString = get(name);
1362        if (valueString == null) {
1363          return defaultValue;
1364        } else {
1365          return StringUtils.getStrings(valueString);
1366        }
1367      }
1368      
1369      /** 
1370       * Get the comma delimited values of the <code>name</code> property as 
1371       * a collection of <code>String</code>s, trimmed of the leading and trailing whitespace.  
1372       * If no such property is specified then empty <code>Collection</code> is returned.
1373       *
1374       * @param name property name.
1375       * @return property value as a collection of <code>String</code>s, or empty <code>Collection</code> 
1376       */
1377      public Collection<String> getTrimmedStringCollection(String name) {
1378        String valueString = get(name);
1379        if (null == valueString) {
1380          Collection<String> empty = new ArrayList<String>();
1381          return empty;
1382        }
1383        return StringUtils.getTrimmedStringCollection(valueString);
1384      }
1385      
1386      /** 
1387       * Get the comma delimited values of the <code>name</code> property as 
1388       * an array of <code>String</code>s, trimmed of the leading and trailing whitespace.
1389       * If no such property is specified then an empty array is returned.
1390       * 
1391       * @param name property name.
1392       * @return property value as an array of trimmed <code>String</code>s, 
1393       *         or empty array. 
1394       */
1395      public String[] getTrimmedStrings(String name) {
1396        String valueString = get(name);
1397        return StringUtils.getTrimmedStrings(valueString);
1398      }
1399    
1400      /** 
1401       * Get the comma delimited values of the <code>name</code> property as 
1402       * an array of <code>String</code>s, trimmed of the leading and trailing whitespace.
1403       * If no such property is specified then default value is returned.
1404       * 
1405       * @param name property name.
1406       * @param defaultValue The default value
1407       * @return property value as an array of trimmed <code>String</code>s, 
1408       *         or default value. 
1409       */
1410      public String[] getTrimmedStrings(String name, String... defaultValue) {
1411        String valueString = get(name);
1412        if (null == valueString) {
1413          return defaultValue;
1414        } else {
1415          return StringUtils.getTrimmedStrings(valueString);
1416        }
1417      }
1418    
1419      /** 
1420       * Set the array of string values for the <code>name</code> property as 
1421       * as comma delimited values.  
1422       * 
1423       * @param name property name.
1424       * @param values The values
1425       */
1426      public void setStrings(String name, String... values) {
1427        set(name, StringUtils.arrayToString(values));
1428      }
1429    
1430      /**
1431       * Get the socket address for <code>name</code> property as a
1432       * <code>InetSocketAddress</code>.
1433       * @param name property name.
1434       * @param defaultAddress the default value
1435       * @param defaultPort the default port
1436       * @return InetSocketAddress
1437       */
1438      public InetSocketAddress getSocketAddr(
1439          String name, String defaultAddress, int defaultPort) {
1440        final String address = get(name, defaultAddress);
1441        return NetUtils.createSocketAddr(address, defaultPort, name);
1442      }
1443    
1444      /**
1445       * Set the socket address for the <code>name</code> property as
1446       * a <code>host:port</code>.
1447       */
1448      public void setSocketAddr(String name, InetSocketAddress addr) {
1449        set(name, NetUtils.getHostPortString(addr));
1450      }
1451      
1452      /**
1453       * Set the socket address a client can use to connect for the
1454       * <code>name</code> property as a <code>host:port</code>.  The wildcard
1455       * address is replaced with the local host's address.
1456       * @param name property name.
1457       * @param addr InetSocketAddress of a listener to store in the given property
1458       * @return InetSocketAddress for clients to connect
1459       */
1460      public InetSocketAddress updateConnectAddr(String name,
1461                                                 InetSocketAddress addr) {
1462        final InetSocketAddress connectAddr = NetUtils.getConnectAddress(addr);
1463        setSocketAddr(name, connectAddr);
1464        return connectAddr;
1465      }
1466      
1467      /**
1468       * Load a class by name.
1469       * 
1470       * @param name the class name.
1471       * @return the class object.
1472       * @throws ClassNotFoundException if the class is not found.
1473       */
1474      public Class<?> getClassByName(String name) throws ClassNotFoundException {
1475        Class<?> ret = getClassByNameOrNull(name);
1476        if (ret == null) {
1477          throw new ClassNotFoundException("Class " + name + " not found");
1478        }
1479        return ret;
1480      }
1481      
1482      /**
1483       * Load a class by name, returning null rather than throwing an exception
1484       * if it couldn't be loaded. This is to avoid the overhead of creating
1485       * an exception.
1486       * 
1487       * @param name the class name
1488       * @return the class object, or null if it could not be found.
1489       */
1490      public Class<?> getClassByNameOrNull(String name) {
1491        Map<String, WeakReference<Class<?>>> map;
1492        
1493        synchronized (CACHE_CLASSES) {
1494          map = CACHE_CLASSES.get(classLoader);
1495          if (map == null) {
1496            map = Collections.synchronizedMap(
1497              new WeakHashMap<String, WeakReference<Class<?>>>());
1498            CACHE_CLASSES.put(classLoader, map);
1499          }
1500        }
1501    
1502        Class<?> clazz = null;
1503        WeakReference<Class<?>> ref = map.get(name); 
1504        if (ref != null) {
1505           clazz = ref.get();
1506        }
1507         
1508        if (clazz == null) {
1509          try {
1510            clazz = Class.forName(name, true, classLoader);
1511          } catch (ClassNotFoundException e) {
1512            // Leave a marker that the class isn't found
1513            map.put(name, new WeakReference<Class<?>>(NEGATIVE_CACHE_SENTINEL));
1514            return null;
1515          }
1516          // two putters can race here, but they'll put the same class
1517          map.put(name, new WeakReference<Class<?>>(clazz));
1518          return clazz;
1519        } else if (clazz == NEGATIVE_CACHE_SENTINEL) {
1520          return null; // not found
1521        } else {
1522          // cache hit
1523          return clazz;
1524        }
1525      }
1526    
1527      /** 
1528       * Get the value of the <code>name</code> property
1529       * as an array of <code>Class</code>.
1530       * The value of the property specifies a list of comma separated class names.  
1531       * If no such property is specified, then <code>defaultValue</code> is 
1532       * returned.
1533       * 
1534       * @param name the property name.
1535       * @param defaultValue default value.
1536       * @return property value as a <code>Class[]</code>, 
1537       *         or <code>defaultValue</code>. 
1538       */
1539      public Class<?>[] getClasses(String name, Class<?> ... defaultValue) {
1540        String[] classnames = getTrimmedStrings(name);
1541        if (classnames == null)
1542          return defaultValue;
1543        try {
1544          Class<?>[] classes = new Class<?>[classnames.length];
1545          for(int i = 0; i < classnames.length; i++) {
1546            classes[i] = getClassByName(classnames[i]);
1547          }
1548          return classes;
1549        } catch (ClassNotFoundException e) {
1550          throw new RuntimeException(e);
1551        }
1552      }
1553    
1554      /** 
1555       * Get the value of the <code>name</code> property as a <code>Class</code>.  
1556       * If no such property is specified, then <code>defaultValue</code> is 
1557       * returned.
1558       * 
1559       * @param name the class name.
1560       * @param defaultValue default value.
1561       * @return property value as a <code>Class</code>, 
1562       *         or <code>defaultValue</code>. 
1563       */
1564      public Class<?> getClass(String name, Class<?> defaultValue) {
1565        String valueString = getTrimmed(name);
1566        if (valueString == null)
1567          return defaultValue;
1568        try {
1569          return getClassByName(valueString);
1570        } catch (ClassNotFoundException e) {
1571          throw new RuntimeException(e);
1572        }
1573      }
1574    
1575      /** 
1576       * Get the value of the <code>name</code> property as a <code>Class</code>
1577       * implementing the interface specified by <code>xface</code>.
1578       *   
1579       * If no such property is specified, then <code>defaultValue</code> is 
1580       * returned.
1581       * 
1582       * An exception is thrown if the returned class does not implement the named
1583       * interface. 
1584       * 
1585       * @param name the class name.
1586       * @param defaultValue default value.
1587       * @param xface the interface implemented by the named class.
1588       * @return property value as a <code>Class</code>, 
1589       *         or <code>defaultValue</code>.
1590       */
1591      public <U> Class<? extends U> getClass(String name, 
1592                                             Class<? extends U> defaultValue, 
1593                                             Class<U> xface) {
1594        try {
1595          Class<?> theClass = getClass(name, defaultValue);
1596          if (theClass != null && !xface.isAssignableFrom(theClass))
1597            throw new RuntimeException(theClass+" not "+xface.getName());
1598          else if (theClass != null)
1599            return theClass.asSubclass(xface);
1600          else
1601            return null;
1602        } catch (Exception e) {
1603          throw new RuntimeException(e);
1604        }
1605      }
1606    
1607      /**
1608       * Get the value of the <code>name</code> property as a <code>List</code>
1609       * of objects implementing the interface specified by <code>xface</code>.
1610       * 
1611       * An exception is thrown if any of the classes does not exist, or if it does
1612       * not implement the named interface.
1613       * 
1614       * @param name the property name.
1615       * @param xface the interface implemented by the classes named by
1616       *        <code>name</code>.
1617       * @return a <code>List</code> of objects implementing <code>xface</code>.
1618       */
1619      @SuppressWarnings("unchecked")
1620      public <U> List<U> getInstances(String name, Class<U> xface) {
1621        List<U> ret = new ArrayList<U>();
1622        Class<?>[] classes = getClasses(name);
1623        for (Class<?> cl: classes) {
1624          if (!xface.isAssignableFrom(cl)) {
1625            throw new RuntimeException(cl + " does not implement " + xface);
1626          }
1627          ret.add((U)ReflectionUtils.newInstance(cl, this));
1628        }
1629        return ret;
1630      }
1631    
1632      /** 
1633       * Set the value of the <code>name</code> property to the name of a 
1634       * <code>theClass</code> implementing the given interface <code>xface</code>.
1635       * 
1636       * An exception is thrown if <code>theClass</code> does not implement the 
1637       * interface <code>xface</code>. 
1638       * 
1639       * @param name property name.
1640       * @param theClass property value.
1641       * @param xface the interface implemented by the named class.
1642       */
1643      public void setClass(String name, Class<?> theClass, Class<?> xface) {
1644        if (!xface.isAssignableFrom(theClass))
1645          throw new RuntimeException(theClass+" not "+xface.getName());
1646        set(name, theClass.getName());
1647      }
1648    
1649      /** 
1650       * Get a local file under a directory named by <i>dirsProp</i> with
1651       * the given <i>path</i>.  If <i>dirsProp</i> contains multiple directories,
1652       * then one is chosen based on <i>path</i>'s hash code.  If the selected
1653       * directory does not exist, an attempt is made to create it.
1654       * 
1655       * @param dirsProp directory in which to locate the file.
1656       * @param path file-path.
1657       * @return local file under the directory with the given path.
1658       */
1659      public Path getLocalPath(String dirsProp, String path)
1660        throws IOException {
1661        String[] dirs = getTrimmedStrings(dirsProp);
1662        int hashCode = path.hashCode();
1663        FileSystem fs = FileSystem.getLocal(this);
1664        for (int i = 0; i < dirs.length; i++) {  // try each local dir
1665          int index = (hashCode+i & Integer.MAX_VALUE) % dirs.length;
1666          Path file = new Path(dirs[index], path);
1667          Path dir = file.getParent();
1668          if (fs.mkdirs(dir) || fs.exists(dir)) {
1669            return file;
1670          }
1671        }
1672        LOG.warn("Could not make " + path + 
1673                 " in local directories from " + dirsProp);
1674        for(int i=0; i < dirs.length; i++) {
1675          int index = (hashCode+i & Integer.MAX_VALUE) % dirs.length;
1676          LOG.warn(dirsProp + "[" + index + "]=" + dirs[index]);
1677        }
1678        throw new IOException("No valid local directories in property: "+dirsProp);
1679      }
1680    
1681      /** 
1682       * Get a local file name under a directory named in <i>dirsProp</i> with
1683       * the given <i>path</i>.  If <i>dirsProp</i> contains multiple directories,
1684       * then one is chosen based on <i>path</i>'s hash code.  If the selected
1685       * directory does not exist, an attempt is made to create it.
1686       * 
1687       * @param dirsProp directory in which to locate the file.
1688       * @param path file-path.
1689       * @return local file under the directory with the given path.
1690       */
1691      public File getFile(String dirsProp, String path)
1692        throws IOException {
1693        String[] dirs = getTrimmedStrings(dirsProp);
1694        int hashCode = path.hashCode();
1695        for (int i = 0; i < dirs.length; i++) {  // try each local dir
1696          int index = (hashCode+i & Integer.MAX_VALUE) % dirs.length;
1697          File file = new File(dirs[index], path);
1698          File dir = file.getParentFile();
1699          if (dir.exists() || dir.mkdirs()) {
1700            return file;
1701          }
1702        }
1703        throw new IOException("No valid local directories in property: "+dirsProp);
1704      }
1705    
1706      /** 
1707       * Get the {@link URL} for the named resource.
1708       * 
1709       * @param name resource name.
1710       * @return the url for the named resource.
1711       */
1712      public URL getResource(String name) {
1713        return classLoader.getResource(name);
1714      }
1715      
1716      /** 
1717       * Get an input stream attached to the configuration resource with the
1718       * given <code>name</code>.
1719       * 
1720       * @param name configuration resource name.
1721       * @return an input stream attached to the resource.
1722       */
1723      public InputStream getConfResourceAsInputStream(String name) {
1724        try {
1725          URL url= getResource(name);
1726    
1727          if (url == null) {
1728            LOG.info(name + " not found");
1729            return null;
1730          } else {
1731            LOG.info("found resource " + name + " at " + url);
1732          }
1733    
1734          return url.openStream();
1735        } catch (Exception e) {
1736          return null;
1737        }
1738      }
1739    
1740      /** 
1741       * Get a {@link Reader} attached to the configuration resource with the
1742       * given <code>name</code>.
1743       * 
1744       * @param name configuration resource name.
1745       * @return a reader attached to the resource.
1746       */
1747      public Reader getConfResourceAsReader(String name) {
1748        try {
1749          URL url= getResource(name);
1750    
1751          if (url == null) {
1752            LOG.info(name + " not found");
1753            return null;
1754          } else {
1755            LOG.info("found resource " + name + " at " + url);
1756          }
1757    
1758          return new InputStreamReader(url.openStream());
1759        } catch (Exception e) {
1760          return null;
1761        }
1762      }
1763    
1764      protected synchronized Properties getProps() {
1765        if (properties == null) {
1766          properties = new Properties();
1767          HashMap<String, String[]> backup = 
1768            new HashMap<String, String[]>(updatingResource);
1769          loadResources(properties, resources, quietmode);
1770          if (overlay!= null) {
1771            properties.putAll(overlay);
1772            for (Map.Entry<Object,Object> item: overlay.entrySet()) {
1773              String key = (String)item.getKey();
1774              updatingResource.put(key, backup.get(key));
1775            }
1776          }
1777        }
1778        return properties;
1779      }
1780    
1781      /**
1782       * Return the number of keys in the configuration.
1783       *
1784       * @return number of keys in the configuration.
1785       */
1786      public int size() {
1787        return getProps().size();
1788      }
1789    
1790      /**
1791       * Clears all keys from the configuration.
1792       */
1793      public void clear() {
1794        getProps().clear();
1795        getOverlay().clear();
1796      }
1797    
1798      /**
1799       * Get an {@link Iterator} to go through the list of <code>String</code> 
1800       * key-value pairs in the configuration.
1801       * 
1802       * @return an iterator over the entries.
1803       */
1804      public Iterator<Map.Entry<String, String>> iterator() {
1805        // Get a copy of just the string to string pairs. After the old object
1806        // methods that allow non-strings to be put into configurations are removed,
1807        // we could replace properties with a Map<String,String> and get rid of this
1808        // code.
1809        Map<String,String> result = new HashMap<String,String>();
1810        for(Map.Entry<Object,Object> item: getProps().entrySet()) {
1811          if (item.getKey() instanceof String && 
1812              item.getValue() instanceof String) {
1813            result.put((String) item.getKey(), (String) item.getValue());
1814          }
1815        }
1816        return result.entrySet().iterator();
1817      }
1818    
1819      private Document parse(DocumentBuilder builder, URL url)
1820          throws IOException, SAXException {
1821        if (!quietmode) {
1822          LOG.info("parsing URL " + url);
1823        }
1824        if (url == null) {
1825          return null;
1826        }
1827        return parse(builder, url.openStream(), url.toString());
1828      }
1829    
1830      private Document parse(DocumentBuilder builder, InputStream is,
1831          String systemId) throws IOException, SAXException {
1832        if (!quietmode) {
1833          LOG.info("parsing input stream " + is);
1834        }
1835        if (is == null) {
1836          return null;
1837        }
1838        try {
1839          return (systemId == null) ? builder.parse(is) : builder.parse(is,
1840              systemId);
1841        } finally {
1842          is.close();
1843        }
1844      }
1845    
1846      private void loadResources(Properties properties,
1847                                 ArrayList<Resource> resources,
1848                                 boolean quiet) {
1849        if(loadDefaults) {
1850          for (String resource : defaultResources) {
1851            loadResource(properties, new Resource(resource), quiet);
1852          }
1853        
1854          //support the hadoop-site.xml as a deprecated case
1855          if(getResource("hadoop-site.xml")!=null) {
1856            loadResource(properties, new Resource("hadoop-site.xml"), quiet);
1857          }
1858        }
1859        
1860        for (int i = 0; i < resources.size(); i++) {
1861          Resource ret = loadResource(properties, resources.get(i), quiet);
1862          if (ret != null) {
1863            resources.set(i, ret);
1864          }
1865        }
1866      }
1867      
1868      private Resource loadResource(Properties properties, Resource wrapper, boolean quiet) {
1869        String name = UNKNOWN_RESOURCE;
1870        try {
1871          Object resource = wrapper.getResource();
1872          name = wrapper.getName();
1873          
1874          DocumentBuilderFactory docBuilderFactory 
1875            = DocumentBuilderFactory.newInstance();
1876          //ignore all comments inside the xml file
1877          docBuilderFactory.setIgnoringComments(true);
1878    
1879          //allow includes in the xml file
1880          docBuilderFactory.setNamespaceAware(true);
1881          try {
1882              docBuilderFactory.setXIncludeAware(true);
1883          } catch (UnsupportedOperationException e) {
1884            LOG.error("Failed to set setXIncludeAware(true) for parser "
1885                    + docBuilderFactory
1886                    + ":" + e,
1887                    e);
1888          }
1889          DocumentBuilder builder = docBuilderFactory.newDocumentBuilder();
1890          Document doc = null;
1891          Element root = null;
1892          boolean returnCachedProperties = false;
1893          
1894          if (resource instanceof URL) {                  // an URL resource
1895            doc = parse(builder, (URL)resource);
1896          } else if (resource instanceof String) {        // a CLASSPATH resource
1897            URL url = getResource((String)resource);
1898            doc = parse(builder, url);
1899          } else if (resource instanceof Path) {          // a file resource
1900            // Can't use FileSystem API or we get an infinite loop
1901            // since FileSystem uses Configuration API.  Use java.io.File instead.
1902            File file = new File(((Path)resource).toUri().getPath())
1903              .getAbsoluteFile();
1904            if (file.exists()) {
1905              if (!quiet) {
1906                LOG.info("parsing File " + file);
1907              }
1908              doc = parse(builder, new BufferedInputStream(
1909                  new FileInputStream(file)), ((Path)resource).toString());
1910            }
1911          } else if (resource instanceof InputStream) {
1912            doc = parse(builder, (InputStream) resource, null);
1913            returnCachedProperties = true;
1914          } else if (resource instanceof Properties) {
1915            overlay(properties, (Properties)resource);
1916          } else if (resource instanceof Element) {
1917            root = (Element)resource;
1918          }
1919    
1920          if (doc == null && root == null) {
1921            if (quiet)
1922              return null;
1923            throw new RuntimeException(resource + " not found");
1924          }
1925    
1926          if (root == null) {
1927            root = doc.getDocumentElement();
1928          }
1929          Properties toAddTo = properties;
1930          if(returnCachedProperties) {
1931            toAddTo = new Properties();
1932          }
1933          if (!"configuration".equals(root.getTagName()))
1934            LOG.fatal("bad conf file: top-level element not <configuration>");
1935          NodeList props = root.getChildNodes();
1936          for (int i = 0; i < props.getLength(); i++) {
1937            Node propNode = props.item(i);
1938            if (!(propNode instanceof Element))
1939              continue;
1940            Element prop = (Element)propNode;
1941            if ("configuration".equals(prop.getTagName())) {
1942              loadResource(toAddTo, new Resource(prop, name), quiet);
1943              continue;
1944            }
1945            if (!"property".equals(prop.getTagName()))
1946              LOG.warn("bad conf file: element not <property>");
1947            NodeList fields = prop.getChildNodes();
1948            String attr = null;
1949            String value = null;
1950            boolean finalParameter = false;
1951            LinkedList<String> source = new LinkedList<String>();
1952            for (int j = 0; j < fields.getLength(); j++) {
1953              Node fieldNode = fields.item(j);
1954              if (!(fieldNode instanceof Element))
1955                continue;
1956              Element field = (Element)fieldNode;
1957              if ("name".equals(field.getTagName()) && field.hasChildNodes())
1958                attr = ((Text)field.getFirstChild()).getData().trim();
1959              if ("value".equals(field.getTagName()) && field.hasChildNodes())
1960                value = ((Text)field.getFirstChild()).getData();
1961              if ("final".equals(field.getTagName()) && field.hasChildNodes())
1962                finalParameter = "true".equals(((Text)field.getFirstChild()).getData());
1963              if ("source".equals(field.getTagName()) && field.hasChildNodes())
1964                source.add(((Text)field.getFirstChild()).getData());
1965            }
1966            source.add(name);
1967            
1968            // Ignore this parameter if it has already been marked as 'final'
1969            if (attr != null) {
1970              if (deprecatedKeyMap.containsKey(attr)) {
1971                DeprecatedKeyInfo keyInfo = deprecatedKeyMap.get(attr);
1972                keyInfo.accessed = false;
1973                for (String key:keyInfo.newKeys) {
1974                  // update new keys with deprecated key's value 
1975                  loadProperty(toAddTo, name, key, value, finalParameter, 
1976                      source.toArray(new String[source.size()]));
1977                }
1978              }
1979              else {
1980                loadProperty(toAddTo, name, attr, value, finalParameter, 
1981                    source.toArray(new String[source.size()]));
1982              }
1983            }
1984          }
1985          
1986          if (returnCachedProperties) {
1987            overlay(properties, toAddTo);
1988            return new Resource(toAddTo, name);
1989          }
1990          return null;
1991        } catch (IOException e) {
1992          LOG.fatal("error parsing conf " + name, e);
1993          throw new RuntimeException(e);
1994        } catch (DOMException e) {
1995          LOG.fatal("error parsing conf " + name, e);
1996          throw new RuntimeException(e);
1997        } catch (SAXException e) {
1998          LOG.fatal("error parsing conf " + name, e);
1999          throw new RuntimeException(e);
2000        } catch (ParserConfigurationException e) {
2001          LOG.fatal("error parsing conf " + name , e);
2002          throw new RuntimeException(e);
2003        }
2004      }
2005    
2006      private void overlay(Properties to, Properties from) {
2007        for (Entry<Object, Object> entry: from.entrySet()) {
2008          to.put(entry.getKey(), entry.getValue());
2009        }
2010      }
2011      
2012      private void loadProperty(Properties properties, String name, String attr,
2013          String value, boolean finalParameter, String[] source) {
2014        if (value != null) {
2015          if (!finalParameters.contains(attr)) {
2016            properties.setProperty(attr, value);
2017            updatingResource.put(attr, source);
2018          } else if (!value.equals(properties.getProperty(attr))) {
2019            LOG.warn(name+":an attempt to override final parameter: "+attr
2020                +";  Ignoring.");
2021          }
2022        }
2023        if (finalParameter) {
2024          finalParameters.add(attr);
2025        }
2026      }
2027    
2028      /** 
2029       * Write out the non-default properties in this configuration to the given
2030       * {@link OutputStream}.
2031       * 
2032       * @param out the output stream to write to.
2033       */
2034      public void writeXml(OutputStream out) throws IOException {
2035        writeXml(new OutputStreamWriter(out));
2036      }
2037    
2038      /** 
2039       * Write out the non-default properties in this configuration to the given
2040       * {@link Writer}.
2041       * 
2042       * @param out the writer to write to.
2043       */
2044      public void writeXml(Writer out) throws IOException {
2045        Document doc = asXmlDocument();
2046    
2047        try {
2048          DOMSource source = new DOMSource(doc);
2049          StreamResult result = new StreamResult(out);
2050          TransformerFactory transFactory = TransformerFactory.newInstance();
2051          Transformer transformer = transFactory.newTransformer();
2052    
2053          // Important to not hold Configuration log while writing result, since
2054          // 'out' may be an HDFS stream which needs to lock this configuration
2055          // from another thread.
2056          transformer.transform(source, result);
2057        } catch (TransformerException te) {
2058          throw new IOException(te);
2059        }
2060      }
2061    
2062      /**
2063       * Return the XML DOM corresponding to this Configuration.
2064       */
2065      private synchronized Document asXmlDocument() throws IOException {
2066        Document doc;
2067        try {
2068          doc =
2069            DocumentBuilderFactory.newInstance().newDocumentBuilder().newDocument();
2070        } catch (ParserConfigurationException pe) {
2071          throw new IOException(pe);
2072        }
2073        Element conf = doc.createElement("configuration");
2074        doc.appendChild(conf);
2075        conf.appendChild(doc.createTextNode("\n"));
2076        handleDeprecation(); //ensure properties is set and deprecation is handled
2077        for (Enumeration e = properties.keys(); e.hasMoreElements();) {
2078          String name = (String)e.nextElement();
2079          Object object = properties.get(name);
2080          String value = null;
2081          if (object instanceof String) {
2082            value = (String) object;
2083          }else {
2084            continue;
2085          }
2086          Element propNode = doc.createElement("property");
2087          conf.appendChild(propNode);
2088    
2089          Element nameNode = doc.createElement("name");
2090          nameNode.appendChild(doc.createTextNode(name));
2091          propNode.appendChild(nameNode);
2092    
2093          Element valueNode = doc.createElement("value");
2094          valueNode.appendChild(doc.createTextNode(value));
2095          propNode.appendChild(valueNode);
2096    
2097          if (updatingResource != null) {
2098            String[] sources = updatingResource.get(name);
2099            if(sources != null) {
2100              for(String s : sources) {
2101                Element sourceNode = doc.createElement("source");
2102                sourceNode.appendChild(doc.createTextNode(s));
2103                propNode.appendChild(sourceNode);
2104              }
2105            }
2106          }
2107          
2108          conf.appendChild(doc.createTextNode("\n"));
2109        }
2110        return doc;
2111      }
2112    
2113      /**
2114       *  Writes out all the parameters and their properties (final and resource) to
2115       *  the given {@link Writer}
2116       *  The format of the output would be 
2117       *  { "properties" : [ {key1,value1,key1.isFinal,key1.resource}, {key2,value2,
2118       *  key2.isFinal,key2.resource}... ] } 
2119       *  It does not output the parameters of the configuration object which is 
2120       *  loaded from an input stream.
2121       * @param out the Writer to write to
2122       * @throws IOException
2123       */
2124      public static void dumpConfiguration(Configuration config,
2125          Writer out) throws IOException {
2126        JsonFactory dumpFactory = new JsonFactory();
2127        JsonGenerator dumpGenerator = dumpFactory.createJsonGenerator(out);
2128        dumpGenerator.writeStartObject();
2129        dumpGenerator.writeFieldName("properties");
2130        dumpGenerator.writeStartArray();
2131        dumpGenerator.flush();
2132        synchronized (config) {
2133          for (Map.Entry<Object,Object> item: config.getProps().entrySet()) {
2134            dumpGenerator.writeStartObject();
2135            dumpGenerator.writeStringField("key", (String) item.getKey());
2136            dumpGenerator.writeStringField("value", 
2137                                           config.get((String) item.getKey()));
2138            dumpGenerator.writeBooleanField("isFinal",
2139                                            config.finalParameters.contains(item.getKey()));
2140            String[] resources = config.updatingResource.get(item.getKey());
2141            String resource = UNKNOWN_RESOURCE;
2142            if(resources != null && resources.length > 0) {
2143              resource = resources[0];
2144            }
2145            dumpGenerator.writeStringField("resource", resource);
2146            dumpGenerator.writeEndObject();
2147          }
2148        }
2149        dumpGenerator.writeEndArray();
2150        dumpGenerator.writeEndObject();
2151        dumpGenerator.flush();
2152      }
2153      
2154      /**
2155       * Get the {@link ClassLoader} for this job.
2156       * 
2157       * @return the correct class loader.
2158       */
2159      public ClassLoader getClassLoader() {
2160        return classLoader;
2161      }
2162      
2163      /**
2164       * Set the class loader that will be used to load the various objects.
2165       * 
2166       * @param classLoader the new class loader.
2167       */
2168      public void setClassLoader(ClassLoader classLoader) {
2169        this.classLoader = classLoader;
2170      }
2171      
2172      @Override
2173      public String toString() {
2174        StringBuilder sb = new StringBuilder();
2175        sb.append("Configuration: ");
2176        if(loadDefaults) {
2177          toString(defaultResources, sb);
2178          if(resources.size()>0) {
2179            sb.append(", ");
2180          }
2181        }
2182        toString(resources, sb);
2183        return sb.toString();
2184      }
2185      
2186      private <T> void toString(List<T> resources, StringBuilder sb) {
2187        ListIterator<T> i = resources.listIterator();
2188        while (i.hasNext()) {
2189          if (i.nextIndex() != 0) {
2190            sb.append(", ");
2191          }
2192          sb.append(i.next());
2193        }
2194      }
2195    
2196      /** 
2197       * Set the quietness-mode. 
2198       * 
2199       * In the quiet-mode, error and informational messages might not be logged.
2200       * 
2201       * @param quietmode <code>true</code> to set quiet-mode on, <code>false</code>
2202       *              to turn it off.
2203       */
2204      public synchronized void setQuietMode(boolean quietmode) {
2205        this.quietmode = quietmode;
2206      }
2207    
2208      synchronized boolean getQuietMode() {
2209        return this.quietmode;
2210      }
2211      
2212      /** For debugging.  List non-default properties to the terminal and exit. */
2213      public static void main(String[] args) throws Exception {
2214        new Configuration().writeXml(System.out);
2215      }
2216    
2217      @Override
2218      public void readFields(DataInput in) throws IOException {
2219        clear();
2220        int size = WritableUtils.readVInt(in);
2221        for(int i=0; i < size; ++i) {
2222          String key = org.apache.hadoop.io.Text.readString(in);
2223          String value = org.apache.hadoop.io.Text.readString(in);
2224          set(key, value); 
2225          String sources[] = WritableUtils.readCompressedStringArray(in);
2226          updatingResource.put(key, sources);
2227        }
2228      }
2229    
2230      //@Override
2231      public void write(DataOutput out) throws IOException {
2232        Properties props = getProps();
2233        WritableUtils.writeVInt(out, props.size());
2234        for(Map.Entry<Object, Object> item: props.entrySet()) {
2235          org.apache.hadoop.io.Text.writeString(out, (String) item.getKey());
2236          org.apache.hadoop.io.Text.writeString(out, (String) item.getValue());
2237          WritableUtils.writeCompressedStringArray(out, 
2238              updatingResource.get(item.getKey()));
2239        }
2240      }
2241      
2242      /**
2243       * get keys matching the the regex 
2244       * @param regex
2245       * @return Map<String,String> with matching keys
2246       */
2247      public Map<String,String> getValByRegex(String regex) {
2248        Pattern p = Pattern.compile(regex);
2249    
2250        Map<String,String> result = new HashMap<String,String>();
2251        Matcher m;
2252    
2253        for(Map.Entry<Object,Object> item: getProps().entrySet()) {
2254          if (item.getKey() instanceof String && 
2255              item.getValue() instanceof String) {
2256            m = p.matcher((String)item.getKey());
2257            if(m.find()) { // match
2258              result.put((String) item.getKey(), (String) item.getValue());
2259            }
2260          }
2261        }
2262        return result;
2263      }
2264    
2265      //Load deprecated keys in common
2266      private static void addDeprecatedKeys() {
2267        Configuration.addDeprecation("topology.script.file.name", 
2268                   new String[]{CommonConfigurationKeys.NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY});
2269        Configuration.addDeprecation("topology.script.number.args", 
2270                   new String[]{CommonConfigurationKeys.NET_TOPOLOGY_SCRIPT_NUMBER_ARGS_KEY});
2271        Configuration.addDeprecation("hadoop.configured.node.mapping", 
2272                   new String[]{CommonConfigurationKeys.NET_TOPOLOGY_CONFIGURED_NODE_MAPPING_KEY});
2273        Configuration.addDeprecation("topology.node.switch.mapping.impl", 
2274                   new String[]{CommonConfigurationKeys.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY});
2275        Configuration.addDeprecation("dfs.df.interval", 
2276                   new String[]{CommonConfigurationKeys.FS_DF_INTERVAL_KEY});
2277        Configuration.addDeprecation("dfs.client.buffer.dir", 
2278                   new String[]{CommonConfigurationKeys.FS_CLIENT_BUFFER_DIR_KEY});
2279        Configuration.addDeprecation("hadoop.native.lib", 
2280                   new String[]{CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_KEY});
2281        Configuration.addDeprecation("fs.default.name", 
2282                   new String[]{CommonConfigurationKeys.FS_DEFAULT_NAME_KEY});
2283      }
2284      
2285      /**
2286       * A unique class which is used as a sentinel value in the caching
2287       * for getClassByName. {@link Configuration#getClassByNameOrNull(String)}
2288       */
2289      private static abstract class NegativeCacheSentinel {}
2290    
2291      public static void dumpDeprecatedKeys() {
2292        for (Map.Entry<String, DeprecatedKeyInfo> entry : deprecatedKeyMap.entrySet()) {
2293          StringBuilder newKeys = new StringBuilder();
2294          for (String newKey : entry.getValue().newKeys) {
2295            newKeys.append(newKey).append("\t");
2296          }
2297          System.out.println(entry.getKey() + "\t" + newKeys.toString());
2298        }
2299      }
2300    }