001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     *
010     *     http://www.apache.org/licenses/LICENSE-2.0
011     *
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    
019    package org.apache.hadoop.mapred;
020    
021    import static org.apache.hadoop.mapreduce.util.CountersStrings.parseEscapedCompactString;
022    import static org.apache.hadoop.mapreduce.util.CountersStrings.toEscapedCompactString;
023    
024    import java.io.DataInput;
025    import java.io.DataOutput;
026    import java.io.IOException;
027    import java.text.ParseException;
028    import java.util.Collection;
029    import java.util.Iterator;
030    
031    import org.apache.commons.collections.IteratorUtils;
032    import org.apache.commons.logging.Log;
033    import org.apache.hadoop.classification.InterfaceAudience;
034    import org.apache.hadoop.classification.InterfaceStability;
035    import org.apache.hadoop.mapreduce.FileSystemCounter;
036    import org.apache.hadoop.mapreduce.counters.AbstractCounterGroup;
037    import org.apache.hadoop.mapreduce.counters.AbstractCounters;
038    import org.apache.hadoop.mapreduce.counters.CounterGroupBase;
039    import org.apache.hadoop.mapreduce.counters.CounterGroupFactory;
040    import org.apache.hadoop.mapreduce.counters.FileSystemCounterGroup;
041    import org.apache.hadoop.mapreduce.counters.FrameworkCounterGroup;
042    import org.apache.hadoop.mapreduce.counters.GenericCounter;
043    import org.apache.hadoop.mapreduce.counters.Limits;
044    import org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter;
045    import org.apache.hadoop.mapreduce.util.CountersStrings;
046    
047    import com.google.common.collect.Iterators;
048    
049    /**
050     * A set of named counters.
051     *
052     * <p><code>Counters</code> represent global counters, defined either by the
053     * Map-Reduce framework or applications. Each <code>Counter</code> can be of
054     * any {@link Enum} type.</p>
055     *
056     * <p><code>Counters</code> are bunched into {@link Group}s, each comprising of
057     * counters from a particular <code>Enum</code> class.
058     */
059    @InterfaceAudience.Public
060    @InterfaceStability.Stable
061    public class Counters
062        extends AbstractCounters<Counters.Counter, Counters.Group> {
063      
064      public static int MAX_COUNTER_LIMIT = Limits.COUNTERS_MAX;
065      
066      public Counters() {
067        super(groupFactory);
068      }
069    
070      public Counters(org.apache.hadoop.mapreduce.Counters newCounters) {
071        super(newCounters, groupFactory);
072      }
073    
074      /**
075       * Downgrade new {@link org.apache.hadoop.mapreduce.Counters} to old Counters
076       * @param newCounters new Counters
077       * @return old Counters instance corresponding to newCounters
078       */
079      static Counters downgrade(org.apache.hadoop.mapreduce.Counters newCounters) {
080        return new Counters(newCounters);
081      }
082    
083      public synchronized Group getGroup(String groupName) {
084        return super.getGroup(groupName);
085      }
086    
087      @SuppressWarnings("unchecked")
088      public synchronized Collection<String> getGroupNames() {
089        return IteratorUtils.toList(super.getGroupNames().iterator());
090      }
091    
092      public synchronized String makeCompactString() {
093        StringBuilder builder = new StringBuilder();
094        boolean first = true;
095        for(Group group: this){
096          for(Counter counter: group) {
097            if (first) {
098              first = false;
099            } else {
100              builder.append(',');
101            }
102            builder.append(group.getDisplayName());
103            builder.append('.');
104            builder.append(counter.getDisplayName());
105            builder.append(':');
106            builder.append(counter.getCounter());
107          }
108        }
109        return builder.toString();
110      }
111      
112      /**
113       * A counter record, comprising its name and value.
114       */
115      @InterfaceAudience.Public
116      @InterfaceStability.Stable
117      public static class Counter implements org.apache.hadoop.mapreduce.Counter {
118        org.apache.hadoop.mapreduce.Counter realCounter;
119    
120        Counter(org.apache.hadoop.mapreduce.Counter counter) {
121          this.realCounter = counter;
122        }
123    
124        public Counter() {
125          this(new GenericCounter());
126        }
127    
128        @SuppressWarnings("deprecation")
129        @Override
130        public void setDisplayName(String displayName) {
131          realCounter.setDisplayName(displayName);
132        }
133    
134        @Override
135        public String getName() {
136          return realCounter.getName();
137        }
138    
139        @Override
140        public String getDisplayName() {
141          return realCounter.getDisplayName();
142        }
143    
144        @Override
145        public long getValue() {
146          return realCounter.getValue();
147        }
148    
149        @Override
150        public void setValue(long value) {
151          realCounter.setValue(value);
152        }
153    
154        @Override
155        public void increment(long incr) {
156          realCounter.increment(incr);
157        }
158    
159        @Override
160        public void write(DataOutput out) throws IOException {
161          realCounter.write(out);
162        }
163    
164        @Override
165        public void readFields(DataInput in) throws IOException {
166          realCounter.readFields(in);
167        }
168    
169        /**
170         * Returns the compact stringified version of the counter in the format
171         * [(actual-name)(display-name)(value)]
172         * @return the stringified result
173         */
174        public String makeEscapedCompactString() {
175          return toEscapedCompactString(realCounter);
176        }
177    
178        /**
179         * Checks for (content) equality of two (basic) counters
180         * @param counter to compare
181         * @return true if content equals
182         * @deprecated
183         */
184        @Deprecated
185        public boolean contentEquals(Counter counter) {
186          return realCounter.equals(counter.getUnderlyingCounter());
187        }
188    
189        /**
190         * @return the value of the counter
191         */
192        public long getCounter() {
193          return realCounter.getValue();
194        }
195    
196        @Override
197        public org.apache.hadoop.mapreduce.Counter getUnderlyingCounter() {
198          return realCounter;
199        }
200        
201        @Override
202        public synchronized boolean equals(Object genericRight) {
203          if (genericRight instanceof Counter) {
204            synchronized (genericRight) {
205              Counter right = (Counter) genericRight;
206              return getName().equals(right.getName()) &&
207                     getDisplayName().equals(right.getDisplayName()) &&
208                     getValue() == right.getValue();
209            }
210          }
211          return false;
212        }
213        
214        @Override
215        public int hashCode() {
216          return realCounter.hashCode();
217        }
218      }
219    
220    
221      /**
222       *  <code>Group</code> of counters, comprising of counters from a particular
223       *  counter {@link Enum} class.
224       *
225       *  <p><code>Group</code>handles localization of the class name and the
226       *  counter names.</p>
227       */
228      @InterfaceAudience.Public
229      @InterfaceStability.Stable
230      public static class Group implements CounterGroupBase<Counter> {
231        private CounterGroupBase<Counter> realGroup;
232        
233        Group(GenericGroup group) {
234          this.realGroup = group;
235        }
236        Group(FSGroupImpl group) {
237          this.realGroup = group;
238        }
239        
240        @SuppressWarnings({ "unchecked", "rawtypes" })
241        Group(FrameworkGroupImpl group) {
242          this.realGroup = group;
243        }
244        
245        /**
246         * @param counterName the name of the counter
247         * @return the value of the specified counter, or 0 if the counter does
248         * not exist.
249         */
250        public long getCounter(String counterName)  {
251          return getCounterValue(realGroup, counterName);
252        }
253    
254        /**
255         * @return the compact stringified version of the group in the format
256         * {(actual-name)(display-name)(value)[][][]} where [] are compact strings
257         * for the counters within.
258         */
259        public String makeEscapedCompactString() {
260          return toEscapedCompactString(realGroup);
261        }
262    
263        /**
264         * Get the counter for the given id and create it if it doesn't exist.
265         * @param id the numeric id of the counter within the group
266         * @param name the internal counter name
267         * @return the counter
268         * @deprecated use {@link #findCounter(String)} instead
269         */
270        @Deprecated
271        public Counter getCounter(int id, String name) {
272          return findCounter(name);
273        }
274    
275        /**
276         * Get the counter for the given name and create it if it doesn't exist.
277         * @param name the internal counter name
278         * @return the counter
279         */
280        public Counter getCounterForName(String name) {
281          return findCounter(name);
282        }
283    
284        @Override
285        public void write(DataOutput out) throws IOException {
286         realGroup.write(out); 
287        }
288    
289        @Override
290        public void readFields(DataInput in) throws IOException {
291          realGroup.readFields(in);
292        }
293    
294        @Override
295        public Iterator<Counter> iterator() {
296          return realGroup.iterator();
297        }
298    
299        @Override
300        public String getName() {
301          return realGroup.getName();
302        }
303    
304        @Override
305        public String getDisplayName() {
306          return realGroup.getDisplayName();
307        }
308    
309        @Override
310        public void setDisplayName(String displayName) {
311          realGroup.setDisplayName(displayName);
312        }
313    
314        @Override
315        public void addCounter(Counter counter) {
316          realGroup.addCounter(counter);
317        }
318    
319        @Override
320        public Counter addCounter(String name, String displayName, long value) {
321          return realGroup.addCounter(name, displayName, value);
322        }
323    
324        @Override
325        public Counter findCounter(String counterName, String displayName) {
326          return realGroup.findCounter(counterName, displayName);
327        }
328    
329        @Override
330        public Counter findCounter(String counterName, boolean create) {
331          return realGroup.findCounter(counterName, create);
332        }
333    
334        @Override
335        public Counter findCounter(String counterName) {
336          return realGroup.findCounter(counterName);
337        }
338    
339        @Override
340        public int size() {
341          return realGroup.size();
342        }
343    
344        @Override
345        public void incrAllCounters(CounterGroupBase<Counter> rightGroup) {
346          realGroup.incrAllCounters(rightGroup);
347        }
348        
349        @Override
350        public CounterGroupBase<Counter> getUnderlyingGroup() {
351          return realGroup;
352        }
353    
354        @Override
355        public synchronized boolean equals(Object genericRight) {
356          if (genericRight instanceof CounterGroupBase<?>) {
357            @SuppressWarnings("unchecked")
358            CounterGroupBase<Counter> right = ((CounterGroupBase<Counter>) 
359            genericRight).getUnderlyingGroup();
360            return Iterators.elementsEqual(iterator(), right.iterator());
361          }
362          return false;
363        }
364    
365        @Override
366        public int hashCode() {
367          return realGroup.hashCode();
368        }
369      }
370    
371      // All the group impls need this for legacy group interface
372      static long getCounterValue(CounterGroupBase<Counter> group, String counterName) {
373        Counter counter = group.findCounter(counterName, false);
374        if (counter != null) return counter.getValue();
375        return 0L;
376      }
377    
378      // Mix the generic group implementation into the Group interface
379      private static class GenericGroup extends AbstractCounterGroup<Counter> {
380    
381        GenericGroup(String name, String displayName, Limits limits) {
382          super(name, displayName, limits);
383        }
384    
385        @Override
386        protected Counter newCounter(String counterName, String displayName,
387                                     long value) {
388          return new Counter(new GenericCounter(counterName, displayName, value));
389        }
390    
391        @Override
392        protected Counter newCounter() {
393          return new Counter();
394        }
395        
396        @Override
397        public CounterGroupBase<Counter> getUnderlyingGroup() {
398         return this;
399        }
400      }
401    
402      // Mix the framework group implementation into the Group interface
403      private static class FrameworkGroupImpl<T extends Enum<T>>
404          extends FrameworkCounterGroup<T, Counter> {
405    
406        FrameworkGroupImpl(Class<T> cls) {
407          super(cls);
408        }
409    
410        @Override
411        protected Counter newCounter(T key) {
412          return new Counter(new FrameworkCounter<T>(key, getName()));
413        }
414    
415        @Override
416        public CounterGroupBase<Counter> getUnderlyingGroup() {
417          return this;
418        }
419      }
420    
421      // Mix the file system counter group implementation into the Group interface
422      private static class FSGroupImpl extends FileSystemCounterGroup<Counter> {
423    
424        @Override
425        protected Counter newCounter(String scheme, FileSystemCounter key) {
426          return new Counter(new FSCounter(scheme, key));
427        }
428    
429        @Override
430        public CounterGroupBase<Counter> getUnderlyingGroup() {
431          return this;
432        }
433      }
434    
435      public synchronized Counter findCounter(String group, String name) {
436        if (name.equals("MAP_INPUT_BYTES")) {
437          LOG.warn("Counter name MAP_INPUT_BYTES is deprecated. " +
438                   "Use FileInputFormatCounters as group name and " +
439                   " BYTES_READ as counter name instead");
440          return findCounter(FileInputFormatCounter.BYTES_READ);
441        }
442        return getGroup(group).getCounterForName(name);
443      }
444    
445      /**
446       * Provide factory methods for counter group factory implementation.
447       * See also the GroupFactory in
448       *  {@link org.apache.hadoop.mapreduce.Counters mapreduce.Counters}
449       */
450      static class GroupFactory extends CounterGroupFactory<Counter, Group> {
451    
452        @Override
453        protected <T extends Enum<T>>
454        FrameworkGroupFactory<Group> newFrameworkGroupFactory(final Class<T> cls) {
455          return new FrameworkGroupFactory<Group>() {
456            @Override public Group newGroup(String name) {
457              return new Group(new FrameworkGroupImpl<T>(cls)); // impl in this package
458            }
459          };
460        }
461    
462        @Override
463        protected Group newGenericGroup(String name, String displayName,
464                                        Limits limits) {
465          return new Group(new GenericGroup(name, displayName, limits));
466        }
467    
468        @Override
469        protected Group newFileSystemGroup() {
470          return new Group(new FSGroupImpl());
471        }
472      }
473    
474      private static final GroupFactory groupFactory = new GroupFactory();
475    
476      /**
477       * Find a counter by using strings
478       * @param group the name of the group
479       * @param id the id of the counter within the group (0 to N-1)
480       * @param name the internal name of the counter
481       * @return the counter for that name
482       * @deprecated use {@link #findCounter(String, String)} instead
483       */
484      @Deprecated
485      public Counter findCounter(String group, int id, String name) {
486        return findCounter(group, name);
487      }
488    
489      /**
490       * Increments the specified counter by the specified amount, creating it if
491       * it didn't already exist.
492       * @param key identifies a counter
493       * @param amount amount by which counter is to be incremented
494       */
495      public void incrCounter(Enum<?> key, long amount) {
496        findCounter(key).increment(amount);
497      }
498    
499      /**
500       * Increments the specified counter by the specified amount, creating it if
501       * it didn't already exist.
502       * @param group the name of the group
503       * @param counter the internal name of the counter
504       * @param amount amount by which counter is to be incremented
505       */
506      public void incrCounter(String group, String counter, long amount) {
507        findCounter(group, counter).increment(amount);
508      }
509    
510      /**
511       * Returns current value of the specified counter, or 0 if the counter
512       * does not exist.
513       * @param key the counter enum to lookup
514       * @return the counter value or 0 if counter not found
515       */
516      public synchronized long getCounter(Enum<?> key) {
517        return findCounter(key).getValue();
518      }
519    
520      /**
521       * Increments multiple counters by their amounts in another Counters
522       * instance.
523       * @param other the other Counters instance
524       */
525      public synchronized void incrAllCounters(Counters other) {
526        for (Group otherGroup: other) {
527          Group group = getGroup(otherGroup.getName());
528          group.setDisplayName(otherGroup.getDisplayName());
529          for (Counter otherCounter : otherGroup) {
530            Counter counter = group.getCounterForName(otherCounter.getName());
531            counter.setDisplayName(otherCounter.getDisplayName());
532            counter.increment(otherCounter.getValue());
533          }
534        }
535      }
536    
537      /**
538       * @return the total number of counters
539       * @deprecated use {@link #countCounters()} instead
540       */
541      public int size() {
542        return countCounters();
543      }
544    
545      /**
546       * Convenience method for computing the sum of two sets of counters.
547       * @param a the first counters
548       * @param b the second counters
549       * @return a new summed counters object
550       */
551      public static Counters sum(Counters a, Counters b) {
552        Counters counters = new Counters();
553        counters.incrAllCounters(a);
554        counters.incrAllCounters(b);
555        return counters;
556      }
557    
558      /**
559       * Logs the current counter values.
560       * @param log The log to use.
561       */
562      public void log(Log log) {
563        log.info("Counters: " + size());
564        for(Group group: this) {
565          log.info("  " + group.getDisplayName());
566          for (Counter counter: group) {
567            log.info("    " + counter.getDisplayName() + "=" +
568                     counter.getCounter());
569          }
570        }
571      }
572    
573      /**
574       * Represent the counter in a textual format that can be converted back to
575       * its object form
576       * @return the string in the following format
577       * {(groupName)(group-displayName)[(counterName)(displayName)(value)][]*}*
578       */
579      public String makeEscapedCompactString() {
580        return toEscapedCompactString(this);
581      }
582    
583      /**
584       * Convert a stringified (by {@link #makeEscapedCompactString()} counter
585       * representation into a counter object.
586       * @param compactString to parse
587       * @return a new counters object
588       * @throws ParseException
589       */
590      public static Counters fromEscapedCompactString(String compactString)
591          throws ParseException {
592        return parseEscapedCompactString(compactString, new Counters());
593      }
594    }