001 /** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 019 package org.apache.hadoop.mapred; 020 021 import java.io.DataInput; 022 import java.io.DataOutput; 023 import java.io.IOException; 024 import java.util.ArrayList; 025 import java.util.Collection; 026 027 import org.apache.hadoop.classification.InterfaceAudience; 028 import org.apache.hadoop.classification.InterfaceStability; 029 import org.apache.hadoop.io.Text; 030 import org.apache.hadoop.io.Writable; 031 import org.apache.hadoop.io.WritableUtils; 032 import org.apache.hadoop.mapreduce.ClusterMetrics; 033 import org.apache.hadoop.mapreduce.TaskTrackerInfo; 034 import org.apache.hadoop.mapreduce.Cluster.JobTrackerStatus; 035 036 /** 037 * Status information on the current state of the Map-Reduce cluster. 038 * 039 * <p><code>ClusterStatus</code> provides clients with information such as: 040 * <ol> 041 * <li> 042 * Size of the cluster. 043 * </li> 044 * <li> 045 * Name of the trackers. 046 * </li> 047 * <li> 048 * Task capacity of the cluster. 049 * </li> 050 * <li> 051 * The number of currently running map & reduce tasks. 052 * </li> 053 * <li> 054 * State of the <code>JobTracker</code>. 055 * </li> 056 * <li> 057 * Details regarding black listed trackers. 058 * </li> 059 * </ol></p> 060 * 061 * <p>Clients can query for the latest <code>ClusterStatus</code>, via 062 * {@link JobClient#getClusterStatus()}.</p> 063 * 064 * @see JobClient 065 */ 066 @InterfaceAudience.Public 067 @InterfaceStability.Stable 068 public class ClusterStatus implements Writable { 069 /** 070 * Class which encapsulates information about a blacklisted tasktracker. 071 * 072 * The information includes the tasktracker's name and reasons for 073 * getting blacklisted. The toString method of the class will print 074 * the information in a whitespace separated fashion to enable parsing. 075 */ 076 public static class BlackListInfo implements Writable { 077 078 private String trackerName; 079 080 private String reasonForBlackListing; 081 082 private String blackListReport; 083 084 BlackListInfo() { 085 } 086 087 088 /** 089 * Gets the blacklisted tasktracker's name. 090 * 091 * @return tracker's name. 092 */ 093 public String getTrackerName() { 094 return trackerName; 095 } 096 097 /** 098 * Gets the reason for which the tasktracker was blacklisted. 099 * 100 * @return reason which tracker was blacklisted 101 */ 102 public String getReasonForBlackListing() { 103 return reasonForBlackListing; 104 } 105 106 /** 107 * Sets the blacklisted tasktracker's name. 108 * 109 * @param trackerName of the tracker. 110 */ 111 void setTrackerName(String trackerName) { 112 this.trackerName = trackerName; 113 } 114 115 /** 116 * Sets the reason for which the tasktracker was blacklisted. 117 * 118 * @param reasonForBlackListing 119 */ 120 void setReasonForBlackListing(String reasonForBlackListing) { 121 this.reasonForBlackListing = reasonForBlackListing; 122 } 123 124 /** 125 * Gets a descriptive report about why the tasktracker was blacklisted. 126 * 127 * @return report describing why the tasktracker was blacklisted. 128 */ 129 public String getBlackListReport() { 130 return blackListReport; 131 } 132 133 /** 134 * Sets a descriptive report about why the tasktracker was blacklisted. 135 * @param blackListReport report describing why the tasktracker 136 * was blacklisted. 137 */ 138 void setBlackListReport(String blackListReport) { 139 this.blackListReport = blackListReport; 140 } 141 142 @Override 143 public void readFields(DataInput in) throws IOException { 144 trackerName = Text.readString(in); 145 reasonForBlackListing = Text.readString(in); 146 blackListReport = Text.readString(in); 147 } 148 149 @Override 150 public void write(DataOutput out) throws IOException { 151 Text.writeString(out, trackerName); 152 Text.writeString(out, reasonForBlackListing); 153 Text.writeString(out, blackListReport); 154 } 155 156 @Override 157 /** 158 * Print information related to the blacklisted tasktracker in a 159 * whitespace separated fashion. 160 * 161 * The method changes any newlines in the report describing why 162 * the tasktracker was blacklisted to a ':' for enabling better 163 * parsing. 164 */ 165 public String toString() { 166 StringBuilder sb = new StringBuilder(); 167 sb.append(trackerName); 168 sb.append("\t"); 169 sb.append(reasonForBlackListing); 170 sb.append("\t"); 171 sb.append(blackListReport.replace("\n", ":")); 172 return sb.toString(); 173 } 174 175 } 176 177 private int numActiveTrackers; 178 private Collection<String> activeTrackers = new ArrayList<String>(); 179 private int numBlacklistedTrackers; 180 private int numExcludedNodes; 181 private long ttExpiryInterval; 182 private int map_tasks; 183 private int reduce_tasks; 184 private int max_map_tasks; 185 private int max_reduce_tasks; 186 private JobTrackerStatus status; 187 private Collection<BlackListInfo> blacklistedTrackersInfo = 188 new ArrayList<BlackListInfo>(); 189 190 ClusterStatus() {} 191 192 /** 193 * Construct a new cluster status. 194 * 195 * @param trackers no. of tasktrackers in the cluster 196 * @param blacklists no of blacklisted task trackers in the cluster 197 * @param ttExpiryInterval the tasktracker expiry interval 198 * @param maps no. of currently running map-tasks in the cluster 199 * @param reduces no. of currently running reduce-tasks in the cluster 200 * @param maxMaps the maximum no. of map tasks in the cluster 201 * @param maxReduces the maximum no. of reduce tasks in the cluster 202 * @param status the {@link JobTrackerStatus} of the <code>JobTracker</code> 203 */ 204 ClusterStatus(int trackers, int blacklists, long ttExpiryInterval, 205 int maps, int reduces, 206 int maxMaps, int maxReduces, JobTrackerStatus status) { 207 this(trackers, blacklists, ttExpiryInterval, maps, reduces, maxMaps, 208 maxReduces, status, 0); 209 } 210 211 /** 212 * Construct a new cluster status. 213 * 214 * @param trackers no. of tasktrackers in the cluster 215 * @param blacklists no of blacklisted task trackers in the cluster 216 * @param ttExpiryInterval the tasktracker expiry interval 217 * @param maps no. of currently running map-tasks in the cluster 218 * @param reduces no. of currently running reduce-tasks in the cluster 219 * @param maxMaps the maximum no. of map tasks in the cluster 220 * @param maxReduces the maximum no. of reduce tasks in the cluster 221 * @param status the {@link JobTrackerStatus} of the <code>JobTracker</code> 222 * @param numDecommissionedNodes number of decommission trackers 223 */ 224 ClusterStatus(int trackers, int blacklists, long ttExpiryInterval, 225 int maps, int reduces, int maxMaps, int maxReduces, 226 JobTrackerStatus status, int numDecommissionedNodes) { 227 numActiveTrackers = trackers; 228 numBlacklistedTrackers = blacklists; 229 this.numExcludedNodes = numDecommissionedNodes; 230 this.ttExpiryInterval = ttExpiryInterval; 231 map_tasks = maps; 232 reduce_tasks = reduces; 233 max_map_tasks = maxMaps; 234 max_reduce_tasks = maxReduces; 235 this.status = status; 236 } 237 238 /** 239 * Construct a new cluster status. 240 * 241 * @param activeTrackers active tasktrackers in the cluster 242 * @param blacklistedTrackers blacklisted tasktrackers in the cluster 243 * @param ttExpiryInterval the tasktracker expiry interval 244 * @param maps no. of currently running map-tasks in the cluster 245 * @param reduces no. of currently running reduce-tasks in the cluster 246 * @param maxMaps the maximum no. of map tasks in the cluster 247 * @param maxReduces the maximum no. of reduce tasks in the cluster 248 * @param status the {@link JobTrackerStatus} of the <code>JobTracker</code> 249 */ 250 ClusterStatus(Collection<String> activeTrackers, 251 Collection<BlackListInfo> blacklistedTrackers, 252 long ttExpiryInterval, 253 int maps, int reduces, int maxMaps, int maxReduces, 254 JobTrackerStatus status) { 255 this(activeTrackers, blacklistedTrackers, ttExpiryInterval, maps, reduces, 256 maxMaps, maxReduces, status, 0); 257 } 258 259 260 /** 261 * Construct a new cluster status. 262 * 263 * @param activeTrackers active tasktrackers in the cluster 264 * @param blackListedTrackerInfo blacklisted tasktrackers information 265 * in the cluster 266 * @param ttExpiryInterval the tasktracker expiry interval 267 * @param maps no. of currently running map-tasks in the cluster 268 * @param reduces no. of currently running reduce-tasks in the cluster 269 * @param maxMaps the maximum no. of map tasks in the cluster 270 * @param maxReduces the maximum no. of reduce tasks in the cluster 271 * @param status the {@link JobTrackerStatus} of the <code>JobTracker</code> 272 * @param numDecommissionNodes number of decommission trackers 273 */ 274 275 ClusterStatus(Collection<String> activeTrackers, 276 Collection<BlackListInfo> blackListedTrackerInfo, long ttExpiryInterval, 277 int maps, int reduces, int maxMaps, int maxReduces, 278 JobTrackerStatus status, int numDecommissionNodes) { 279 this(activeTrackers.size(), blackListedTrackerInfo.size(), 280 ttExpiryInterval, maps, reduces, maxMaps, maxReduces, status, 281 numDecommissionNodes); 282 this.activeTrackers = activeTrackers; 283 this.blacklistedTrackersInfo = blackListedTrackerInfo; 284 } 285 286 /** 287 * Get the number of task trackers in the cluster. 288 * 289 * @return the number of task trackers in the cluster. 290 */ 291 public int getTaskTrackers() { 292 return numActiveTrackers; 293 } 294 295 /** 296 * Get the names of task trackers in the cluster. 297 * 298 * @return the active task trackers in the cluster. 299 */ 300 public Collection<String> getActiveTrackerNames() { 301 return activeTrackers; 302 } 303 304 /** 305 * Get the names of task trackers in the cluster. 306 * 307 * @return the blacklisted task trackers in the cluster. 308 */ 309 public Collection<String> getBlacklistedTrackerNames() { 310 ArrayList<String> blacklistedTrackers = new ArrayList<String>(); 311 for(BlackListInfo bi : blacklistedTrackersInfo) { 312 blacklistedTrackers.add(bi.getTrackerName()); 313 } 314 return blacklistedTrackers; 315 } 316 317 /** 318 * Get the number of blacklisted task trackers in the cluster. 319 * 320 * @return the number of blacklisted task trackers in the cluster. 321 */ 322 public int getBlacklistedTrackers() { 323 return numBlacklistedTrackers; 324 } 325 326 /** 327 * Get the number of excluded hosts in the cluster. 328 * @return the number of excluded hosts in the cluster. 329 */ 330 public int getNumExcludedNodes() { 331 return numExcludedNodes; 332 } 333 334 /** 335 * Get the tasktracker expiry interval for the cluster 336 * @return the expiry interval in msec 337 */ 338 public long getTTExpiryInterval() { 339 return ttExpiryInterval; 340 } 341 342 /** 343 * Get the number of currently running map tasks in the cluster. 344 * 345 * @return the number of currently running map tasks in the cluster. 346 */ 347 public int getMapTasks() { 348 return map_tasks; 349 } 350 351 /** 352 * Get the number of currently running reduce tasks in the cluster. 353 * 354 * @return the number of currently running reduce tasks in the cluster. 355 */ 356 public int getReduceTasks() { 357 return reduce_tasks; 358 } 359 360 /** 361 * Get the maximum capacity for running map tasks in the cluster. 362 * 363 * @return the maximum capacity for running map tasks in the cluster. 364 */ 365 public int getMaxMapTasks() { 366 return max_map_tasks; 367 } 368 369 /** 370 * Get the maximum capacity for running reduce tasks in the cluster. 371 * 372 * @return the maximum capacity for running reduce tasks in the cluster. 373 */ 374 public int getMaxReduceTasks() { 375 return max_reduce_tasks; 376 } 377 378 /** 379 * Get the JobTracker's status. 380 * 381 * @return {@link JobTrackerStatus} of the JobTracker 382 */ 383 public JobTrackerStatus getJobTrackerStatus() { 384 return status; 385 } 386 387 /** 388 * Gets the list of blacklisted trackers along with reasons for blacklisting. 389 * 390 * @return the collection of {@link BlackListInfo} objects. 391 * 392 */ 393 public Collection<BlackListInfo> getBlackListedTrackersInfo() { 394 return blacklistedTrackersInfo; 395 } 396 397 public void write(DataOutput out) throws IOException { 398 if (activeTrackers.size() == 0) { 399 out.writeInt(numActiveTrackers); 400 out.writeInt(0); 401 } else { 402 out.writeInt(activeTrackers.size()); 403 out.writeInt(activeTrackers.size()); 404 for (String tracker : activeTrackers) { 405 Text.writeString(out, tracker); 406 } 407 } 408 if (blacklistedTrackersInfo.size() == 0) { 409 out.writeInt(numBlacklistedTrackers); 410 out.writeInt(blacklistedTrackersInfo.size()); 411 } else { 412 out.writeInt(blacklistedTrackersInfo.size()); 413 out.writeInt(blacklistedTrackersInfo.size()); 414 for (BlackListInfo tracker : blacklistedTrackersInfo) { 415 tracker.write(out); 416 } 417 } 418 out.writeInt(numExcludedNodes); 419 out.writeLong(ttExpiryInterval); 420 out.writeInt(map_tasks); 421 out.writeInt(reduce_tasks); 422 out.writeInt(max_map_tasks); 423 out.writeInt(max_reduce_tasks); 424 WritableUtils.writeEnum(out, status); 425 } 426 427 public void readFields(DataInput in) throws IOException { 428 numActiveTrackers = in.readInt(); 429 int numTrackerNames = in.readInt(); 430 if (numTrackerNames > 0) { 431 for (int i = 0; i < numTrackerNames; i++) { 432 String name = Text.readString(in); 433 activeTrackers.add(name); 434 } 435 } 436 numBlacklistedTrackers = in.readInt(); 437 int blackListTrackerInfoSize = in.readInt(); 438 if(blackListTrackerInfoSize > 0) { 439 for (int i = 0; i < blackListTrackerInfoSize; i++) { 440 BlackListInfo info = new BlackListInfo(); 441 info.readFields(in); 442 blacklistedTrackersInfo.add(info); 443 } 444 } 445 numExcludedNodes = in.readInt(); 446 ttExpiryInterval = in.readLong(); 447 map_tasks = in.readInt(); 448 reduce_tasks = in.readInt(); 449 max_map_tasks = in.readInt(); 450 max_reduce_tasks = in.readInt(); 451 status = WritableUtils.readEnum(in, JobTrackerStatus.class); 452 } 453 }