001 /** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 package org.apache.hadoop.fs.http.server; 019 020 import org.apache.hadoop.classification.InterfaceAudience; 021 import org.apache.hadoop.fs.ContentSummary; 022 import org.apache.hadoop.fs.FileChecksum; 023 import org.apache.hadoop.fs.FileStatus; 024 import org.apache.hadoop.fs.FileSystem; 025 import org.apache.hadoop.fs.GlobFilter; 026 import org.apache.hadoop.fs.Path; 027 import org.apache.hadoop.fs.PathFilter; 028 import org.apache.hadoop.fs.http.client.HttpFSFileSystem; 029 import org.apache.hadoop.fs.permission.FsPermission; 030 import org.apache.hadoop.io.IOUtils; 031 import org.apache.hadoop.lib.service.FileSystemAccess; 032 import org.json.simple.JSONArray; 033 import org.json.simple.JSONObject; 034 035 import java.io.IOException; 036 import java.io.InputStream; 037 import java.io.OutputStream; 038 import java.util.LinkedHashMap; 039 import java.util.Map; 040 041 /** 042 * FileSystem operation executors used by {@link HttpFSServer}. 043 */ 044 @InterfaceAudience.Private 045 public class FSOperations { 046 047 @SuppressWarnings({"unchecked", "deprecation"}) 048 private static Map fileStatusToJSONRaw(FileStatus status, boolean emptyPathSuffix) { 049 Map json = new LinkedHashMap(); 050 json.put(HttpFSFileSystem.PATH_SUFFIX_JSON, (emptyPathSuffix) ? "" : status.getPath().getName()); 051 json.put(HttpFSFileSystem.TYPE_JSON, HttpFSFileSystem.FILE_TYPE.getType(status).toString()); 052 json.put(HttpFSFileSystem.LENGTH_JSON, status.getLen()); 053 json.put(HttpFSFileSystem.OWNER_JSON, status.getOwner()); 054 json.put(HttpFSFileSystem.GROUP_JSON, status.getGroup()); 055 json.put(HttpFSFileSystem.PERMISSION_JSON, HttpFSFileSystem.permissionToString(status.getPermission())); 056 json.put(HttpFSFileSystem.ACCESS_TIME_JSON, status.getAccessTime()); 057 json.put(HttpFSFileSystem.MODIFICATION_TIME_JSON, status.getModificationTime()); 058 json.put(HttpFSFileSystem.BLOCK_SIZE_JSON, status.getBlockSize()); 059 json.put(HttpFSFileSystem.REPLICATION_JSON, status.getReplication()); 060 return json; 061 } 062 063 /** 064 * Converts a FileSystemAccess <code>FileStatus</code> object into a JSON 065 * object. 066 * 067 * @param status FileSystemAccess file status. 068 * 069 * @return The JSON representation of the file status. 070 */ 071 @SuppressWarnings({"unchecked", "deprecation"}) 072 private static Map fileStatusToJSON(FileStatus status) { 073 Map json = new LinkedHashMap(); 074 json.put(HttpFSFileSystem.FILE_STATUS_JSON, fileStatusToJSONRaw(status, true)); 075 return json; 076 } 077 078 /** 079 * Converts a <code>FileChecksum</code> object into a JSON array 080 * object. 081 * 082 * @param checksum file checksum. 083 * 084 * @return The JSON representation of the file checksum. 085 */ 086 @SuppressWarnings({"unchecked"}) 087 private static Map fileChecksumToJSON(FileChecksum checksum) { 088 Map json = new LinkedHashMap(); 089 json.put(HttpFSFileSystem.CHECKSUM_ALGORITHM_JSON, checksum.getAlgorithmName()); 090 json.put(HttpFSFileSystem.CHECKSUM_BYTES_JSON, 091 org.apache.hadoop.util.StringUtils.byteToHexString(checksum.getBytes())); 092 json.put(HttpFSFileSystem.CHECKSUM_LENGTH_JSON, checksum.getLength()); 093 Map response = new LinkedHashMap(); 094 response.put(HttpFSFileSystem.FILE_CHECKSUM_JSON, json); 095 return response; 096 } 097 098 /** 099 * Converts a <code>ContentSummary</code> object into a JSON array 100 * object. 101 * 102 * @param contentSummary the content summary 103 * 104 * @return The JSON representation of the content summary. 105 */ 106 @SuppressWarnings({"unchecked"}) 107 private static Map contentSummaryToJSON(ContentSummary contentSummary) { 108 Map json = new LinkedHashMap(); 109 json.put(HttpFSFileSystem.CONTENT_SUMMARY_DIRECTORY_COUNT_JSON, contentSummary.getDirectoryCount()); 110 json.put(HttpFSFileSystem.CONTENT_SUMMARY_FILE_COUNT_JSON, contentSummary.getFileCount()); 111 json.put(HttpFSFileSystem.CONTENT_SUMMARY_LENGTH_JSON, contentSummary.getLength()); 112 json.put(HttpFSFileSystem.CONTENT_SUMMARY_QUOTA_JSON, contentSummary.getQuota()); 113 json.put(HttpFSFileSystem.CONTENT_SUMMARY_SPACE_CONSUMED_JSON, contentSummary.getSpaceConsumed()); 114 json.put(HttpFSFileSystem.CONTENT_SUMMARY_SPACE_QUOTA_JSON, contentSummary.getSpaceQuota()); 115 Map response = new LinkedHashMap(); 116 response.put(HttpFSFileSystem.CONTENT_SUMMARY_JSON, json); 117 return response; 118 } 119 120 /** 121 * Converts a FileSystemAccess <code>FileStatus</code> array into a JSON array 122 * object. 123 * 124 * @param status FileSystemAccess file status array. 125 * <code>SCHEME://HOST:PORT</code> in the file status. 126 * 127 * @return The JSON representation of the file status array. 128 */ 129 @SuppressWarnings("unchecked") 130 private static Map fileStatusToJSON(FileStatus[] status) { 131 JSONArray json = new JSONArray(); 132 if (status != null) { 133 for (FileStatus s : status) { 134 json.add(fileStatusToJSONRaw(s, false)); 135 } 136 } 137 Map response = new LinkedHashMap(); 138 Map temp = new LinkedHashMap(); 139 temp.put(HttpFSFileSystem.FILE_STATUS_JSON, json); 140 response.put(HttpFSFileSystem.FILE_STATUSES_JSON, temp); 141 return response; 142 } 143 144 /** 145 * Converts an object into a Json Map with with one key-value entry. 146 * <p/> 147 * It assumes the given value is either a JSON primitive type or a 148 * <code>JsonAware</code> instance. 149 * 150 * @param name name for the key of the entry. 151 * @param value for the value of the entry. 152 * 153 * @return the JSON representation of the key-value pair. 154 */ 155 @SuppressWarnings("unchecked") 156 private static JSONObject toJSON(String name, Object value) { 157 JSONObject json = new JSONObject(); 158 json.put(name, value); 159 return json; 160 } 161 162 /** 163 * Executor that performs an append FileSystemAccess files system operation. 164 */ 165 @InterfaceAudience.Private 166 public static class FSAppend implements FileSystemAccess.FileSystemExecutor<Void> { 167 private InputStream is; 168 private Path path; 169 170 /** 171 * Creates an Append executor. 172 * 173 * @param is input stream to append. 174 * @param path path of the file to append. 175 */ 176 public FSAppend(InputStream is, String path) { 177 this.is = is; 178 this.path = new Path(path); 179 } 180 181 /** 182 * Executes the filesystem operation. 183 * 184 * @param fs filesystem instance to use. 185 * 186 * @return void. 187 * 188 * @throws IOException thrown if an IO error occured. 189 */ 190 @Override 191 public Void execute(FileSystem fs) throws IOException { 192 int bufferSize = fs.getConf().getInt("httpfs.buffer.size", 4096); 193 OutputStream os = fs.append(path, bufferSize); 194 IOUtils.copyBytes(is, os, bufferSize, true); 195 os.close(); 196 return null; 197 } 198 199 } 200 201 /** 202 * Executor that performs a content-summary FileSystemAccess files system operation. 203 */ 204 @InterfaceAudience.Private 205 public static class FSContentSummary implements FileSystemAccess.FileSystemExecutor<Map> { 206 private Path path; 207 208 /** 209 * Creates a content-summary executor. 210 * 211 * @param path the path to retrieve the content-summary. 212 */ 213 public FSContentSummary(String path) { 214 this.path = new Path(path); 215 } 216 217 /** 218 * Executes the filesystem operation. 219 * 220 * @param fs filesystem instance to use. 221 * 222 * @return a Map object (JSON friendly) with the content-summary. 223 * 224 * @throws IOException thrown if an IO error occured. 225 */ 226 @Override 227 public Map execute(FileSystem fs) throws IOException { 228 ContentSummary contentSummary = fs.getContentSummary(path); 229 return contentSummaryToJSON(contentSummary); 230 } 231 232 } 233 234 /** 235 * Executor that performs a create FileSystemAccess files system operation. 236 */ 237 @InterfaceAudience.Private 238 public static class FSCreate implements FileSystemAccess.FileSystemExecutor<Void> { 239 private InputStream is; 240 private Path path; 241 private short permission; 242 private boolean override; 243 private short replication; 244 private long blockSize; 245 246 /** 247 * Creates a Create executor. 248 * 249 * @param is input stream to for the file to create. 250 * @param path path of the file to create. 251 * @param perm permission for the file. 252 * @param override if the file should be overriden if it already exist. 253 * @param repl the replication factor for the file. 254 * @param blockSize the block size for the file. 255 */ 256 public FSCreate(InputStream is, String path, short perm, boolean override, 257 short repl, long blockSize) { 258 this.is = is; 259 this.path = new Path(path); 260 this.permission = perm; 261 this.override = override; 262 this.replication = repl; 263 this.blockSize = blockSize; 264 } 265 266 /** 267 * Executes the filesystem operation. 268 * 269 * @param fs filesystem instance to use. 270 * 271 * @return The URI of the created file. 272 * 273 * @throws IOException thrown if an IO error occured. 274 */ 275 @Override 276 public Void execute(FileSystem fs) throws IOException { 277 if (replication == -1) { 278 replication = fs.getDefaultReplication(path); 279 } 280 if (blockSize == -1) { 281 blockSize = fs.getDefaultBlockSize(path); 282 } 283 FsPermission fsPermission = new FsPermission(permission); 284 int bufferSize = fs.getConf().getInt("httpfs.buffer.size", 4096); 285 OutputStream os = fs.create(path, fsPermission, override, bufferSize, replication, blockSize, null); 286 IOUtils.copyBytes(is, os, bufferSize, true); 287 os.close(); 288 return null; 289 } 290 291 } 292 293 /** 294 * Executor that performs a delete FileSystemAccess files system operation. 295 */ 296 @InterfaceAudience.Private 297 public static class FSDelete implements FileSystemAccess.FileSystemExecutor<JSONObject> { 298 private Path path; 299 private boolean recursive; 300 301 /** 302 * Creates a Delete executor. 303 * 304 * @param path path to delete. 305 * @param recursive if the delete should be recursive or not. 306 */ 307 public FSDelete(String path, boolean recursive) { 308 this.path = new Path(path); 309 this.recursive = recursive; 310 } 311 312 /** 313 * Executes the filesystem operation. 314 * 315 * @param fs filesystem instance to use. 316 * 317 * @return <code>true</code> if the delete operation was successful, 318 * <code>false</code> otherwise. 319 * 320 * @throws IOException thrown if an IO error occured. 321 */ 322 @Override 323 public JSONObject execute(FileSystem fs) throws IOException { 324 boolean deleted = fs.delete(path, recursive); 325 return toJSON(HttpFSFileSystem.DELETE_JSON.toLowerCase(), deleted); 326 } 327 328 } 329 330 /** 331 * Executor that performs a file-checksum FileSystemAccess files system operation. 332 */ 333 @InterfaceAudience.Private 334 public static class FSFileChecksum implements FileSystemAccess.FileSystemExecutor<Map> { 335 private Path path; 336 337 /** 338 * Creates a file-checksum executor. 339 * 340 * @param path the path to retrieve the checksum. 341 */ 342 public FSFileChecksum(String path) { 343 this.path = new Path(path); 344 } 345 346 /** 347 * Executes the filesystem operation. 348 * 349 * @param fs filesystem instance to use. 350 * 351 * @return a Map object (JSON friendly) with the file checksum. 352 * 353 * @throws IOException thrown if an IO error occured. 354 */ 355 @Override 356 public Map execute(FileSystem fs) throws IOException { 357 FileChecksum checksum = fs.getFileChecksum(path); 358 return fileChecksumToJSON(checksum); 359 } 360 361 } 362 363 /** 364 * Executor that performs a file-status FileSystemAccess files system operation. 365 */ 366 @InterfaceAudience.Private 367 public static class FSFileStatus implements FileSystemAccess.FileSystemExecutor<Map> { 368 private Path path; 369 370 /** 371 * Creates a file-status executor. 372 * 373 * @param path the path to retrieve the status. 374 */ 375 public FSFileStatus(String path) { 376 this.path = new Path(path); 377 } 378 379 /** 380 * Executes the filesystem operation. 381 * 382 * @param fs filesystem instance to use. 383 * 384 * @return a Map object (JSON friendly) with the file status. 385 * 386 * @throws IOException thrown if an IO error occured. 387 */ 388 @Override 389 public Map execute(FileSystem fs) throws IOException { 390 FileStatus status = fs.getFileStatus(path); 391 return fileStatusToJSON(status); 392 } 393 394 } 395 396 /** 397 * Executor that performs a home-dir FileSystemAccess files system operation. 398 */ 399 @InterfaceAudience.Private 400 public static class FSHomeDir implements FileSystemAccess.FileSystemExecutor<JSONObject> { 401 402 /** 403 * Executes the filesystem operation. 404 * 405 * @param fs filesystem instance to use. 406 * 407 * @return a JSON object with the user home directory. 408 * 409 * @throws IOException thrown if an IO error occured. 410 */ 411 @Override 412 @SuppressWarnings("unchecked") 413 public JSONObject execute(FileSystem fs) throws IOException { 414 Path homeDir = fs.getHomeDirectory(); 415 JSONObject json = new JSONObject(); 416 json.put(HttpFSFileSystem.HOME_DIR_JSON, homeDir.toUri().getPath()); 417 return json; 418 } 419 420 } 421 422 /** 423 * Executor that performs a list-status FileSystemAccess files system operation. 424 */ 425 @InterfaceAudience.Private 426 public static class FSListStatus implements FileSystemAccess.FileSystemExecutor<Map>, PathFilter { 427 private Path path; 428 private PathFilter filter; 429 430 /** 431 * Creates a list-status executor. 432 * 433 * @param path the directory to retrieve the status of its contents. 434 * @param filter glob filter to use. 435 * 436 * @throws IOException thrown if the filter expression is incorrect. 437 */ 438 public FSListStatus(String path, String filter) throws IOException { 439 this.path = new Path(path); 440 this.filter = (filter == null) ? this : new GlobFilter(filter); 441 } 442 443 /** 444 * Executes the filesystem operation. 445 * 446 * @param fs filesystem instance to use. 447 * 448 * @return a Map with the file status of the directory 449 * contents. 450 * 451 * @throws IOException thrown if an IO error occured. 452 */ 453 @Override 454 public Map execute(FileSystem fs) throws IOException { 455 FileStatus[] status = fs.listStatus(path, filter); 456 return fileStatusToJSON(status); 457 } 458 459 @Override 460 public boolean accept(Path path) { 461 return true; 462 } 463 464 } 465 466 /** 467 * Executor that performs a mkdirs FileSystemAccess files system operation. 468 */ 469 @InterfaceAudience.Private 470 public static class FSMkdirs implements FileSystemAccess.FileSystemExecutor<JSONObject> { 471 472 private Path path; 473 private short permission; 474 475 /** 476 * Creates a mkdirs executor. 477 * 478 * @param path directory path to create. 479 * @param permission permission to use. 480 */ 481 public FSMkdirs(String path, short permission) { 482 this.path = new Path(path); 483 this.permission = permission; 484 } 485 486 /** 487 * Executes the filesystem operation. 488 * 489 * @param fs filesystem instance to use. 490 * 491 * @return <code>true</code> if the mkdirs operation was successful, 492 * <code>false</code> otherwise. 493 * 494 * @throws IOException thrown if an IO error occured. 495 */ 496 @Override 497 public JSONObject execute(FileSystem fs) throws IOException { 498 FsPermission fsPermission = new FsPermission(permission); 499 boolean mkdirs = fs.mkdirs(path, fsPermission); 500 return toJSON(HttpFSFileSystem.MKDIRS_JSON, mkdirs); 501 } 502 503 } 504 505 /** 506 * Executor that performs a open FileSystemAccess files system operation. 507 */ 508 @InterfaceAudience.Private 509 public static class FSOpen implements FileSystemAccess.FileSystemExecutor<InputStream> { 510 private Path path; 511 512 /** 513 * Creates a open executor. 514 * 515 * @param path file to open. 516 */ 517 public FSOpen(String path) { 518 this.path = new Path(path); 519 } 520 521 /** 522 * Executes the filesystem operation. 523 * 524 * @param fs filesystem instance to use. 525 * 526 * @return The inputstream of the file. 527 * 528 * @throws IOException thrown if an IO error occured. 529 */ 530 @Override 531 public InputStream execute(FileSystem fs) throws IOException { 532 int bufferSize = HttpFSServerWebApp.get().getConfig().getInt("httpfs.buffer.size", 4096); 533 return fs.open(path, bufferSize); 534 } 535 536 } 537 538 /** 539 * Executor that performs a rename FileSystemAccess files system operation. 540 */ 541 @InterfaceAudience.Private 542 public static class FSRename implements FileSystemAccess.FileSystemExecutor<JSONObject> { 543 private Path path; 544 private Path toPath; 545 546 /** 547 * Creates a rename executor. 548 * 549 * @param path path to rename. 550 * @param toPath new name. 551 */ 552 public FSRename(String path, String toPath) { 553 this.path = new Path(path); 554 this.toPath = new Path(toPath); 555 } 556 557 /** 558 * Executes the filesystem operation. 559 * 560 * @param fs filesystem instance to use. 561 * 562 * @return <code>true</code> if the rename operation was successful, 563 * <code>false</code> otherwise. 564 * 565 * @throws IOException thrown if an IO error occured. 566 */ 567 @Override 568 public JSONObject execute(FileSystem fs) throws IOException { 569 boolean renamed = fs.rename(path, toPath); 570 return toJSON(HttpFSFileSystem.RENAME_JSON, renamed); 571 } 572 573 } 574 575 /** 576 * Executor that performs a set-owner FileSystemAccess files system operation. 577 */ 578 @InterfaceAudience.Private 579 public static class FSSetOwner implements FileSystemAccess.FileSystemExecutor<Void> { 580 private Path path; 581 private String owner; 582 private String group; 583 584 /** 585 * Creates a set-owner executor. 586 * 587 * @param path the path to set the owner. 588 * @param owner owner to set. 589 * @param group group to set. 590 */ 591 public FSSetOwner(String path, String owner, String group) { 592 this.path = new Path(path); 593 this.owner = owner; 594 this.group = group; 595 } 596 597 /** 598 * Executes the filesystem operation. 599 * 600 * @param fs filesystem instance to use. 601 * 602 * @return void. 603 * 604 * @throws IOException thrown if an IO error occured. 605 */ 606 @Override 607 public Void execute(FileSystem fs) throws IOException { 608 fs.setOwner(path, owner, group); 609 return null; 610 } 611 612 } 613 614 /** 615 * Executor that performs a set-permission FileSystemAccess files system operation. 616 */ 617 @InterfaceAudience.Private 618 public static class FSSetPermission implements FileSystemAccess.FileSystemExecutor<Void> { 619 620 private Path path; 621 private short permission; 622 623 /** 624 * Creates a set-permission executor. 625 * 626 * @param path path to set the permission. 627 * @param permission permission to set. 628 */ 629 public FSSetPermission(String path, short permission) { 630 this.path = new Path(path); 631 this.permission = permission; 632 } 633 634 /** 635 * Executes the filesystem operation. 636 * 637 * @param fs filesystem instance to use. 638 * 639 * @return void. 640 * 641 * @throws IOException thrown if an IO error occured. 642 */ 643 @Override 644 public Void execute(FileSystem fs) throws IOException { 645 FsPermission fsPermission = new FsPermission(permission); 646 fs.setPermission(path, fsPermission); 647 return null; 648 } 649 650 } 651 652 /** 653 * Executor that performs a set-replication FileSystemAccess files system operation. 654 */ 655 @InterfaceAudience.Private 656 public static class FSSetReplication implements FileSystemAccess.FileSystemExecutor<JSONObject> { 657 private Path path; 658 private short replication; 659 660 /** 661 * Creates a set-replication executor. 662 * 663 * @param path path to set the replication factor. 664 * @param replication replication factor to set. 665 */ 666 public FSSetReplication(String path, short replication) { 667 this.path = new Path(path); 668 this.replication = replication; 669 } 670 671 /** 672 * Executes the filesystem operation. 673 * 674 * @param fs filesystem instance to use. 675 * 676 * @return <code>true</code> if the replication value was set, 677 * <code>false</code> otherwise. 678 * 679 * @throws IOException thrown if an IO error occured. 680 */ 681 @Override 682 @SuppressWarnings("unchecked") 683 public JSONObject execute(FileSystem fs) throws IOException { 684 boolean ret = fs.setReplication(path, replication); 685 JSONObject json = new JSONObject(); 686 json.put(HttpFSFileSystem.SET_REPLICATION_JSON, ret); 687 return json; 688 } 689 690 } 691 692 /** 693 * Executor that performs a set-times FileSystemAccess files system operation. 694 */ 695 @InterfaceAudience.Private 696 public static class FSSetTimes implements FileSystemAccess.FileSystemExecutor<Void> { 697 private Path path; 698 private long mTime; 699 private long aTime; 700 701 /** 702 * Creates a set-times executor. 703 * 704 * @param path path to set the times. 705 * @param mTime modified time to set. 706 * @param aTime access time to set. 707 */ 708 public FSSetTimes(String path, long mTime, long aTime) { 709 this.path = new Path(path); 710 this.mTime = mTime; 711 this.aTime = aTime; 712 } 713 714 /** 715 * Executes the filesystem operation. 716 * 717 * @param fs filesystem instance to use. 718 * 719 * @return void. 720 * 721 * @throws IOException thrown if an IO error occured. 722 */ 723 @Override 724 public Void execute(FileSystem fs) throws IOException { 725 fs.setTimes(path, mTime, aTime); 726 return null; 727 } 728 729 } 730 731 }