001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     *
010     *     http://www.apache.org/licenses/LICENSE-2.0
011     *
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    package org.apache.hadoop.fs.http.server;
019    
020    import org.apache.hadoop.classification.InterfaceAudience;
021    import org.apache.hadoop.fs.ContentSummary;
022    import org.apache.hadoop.fs.FileChecksum;
023    import org.apache.hadoop.fs.FileStatus;
024    import org.apache.hadoop.fs.FileSystem;
025    import org.apache.hadoop.fs.GlobFilter;
026    import org.apache.hadoop.fs.Path;
027    import org.apache.hadoop.fs.PathFilter;
028    import org.apache.hadoop.fs.http.client.HttpFSFileSystem;
029    import org.apache.hadoop.fs.permission.FsPermission;
030    import org.apache.hadoop.io.IOUtils;
031    import org.apache.hadoop.lib.service.FileSystemAccess;
032    import org.json.simple.JSONArray;
033    import org.json.simple.JSONObject;
034    
035    import java.io.IOException;
036    import java.io.InputStream;
037    import java.io.OutputStream;
038    import java.util.LinkedHashMap;
039    import java.util.Map;
040    
041    /**
042     * FileSystem operation executors used by {@link HttpFSServer}.
043     */
044    @InterfaceAudience.Private
045    public class FSOperations {
046    
047      @SuppressWarnings({"unchecked", "deprecation"})
048      private static Map fileStatusToJSONRaw(FileStatus status, boolean emptyPathSuffix) {
049        Map json = new LinkedHashMap();
050        json.put(HttpFSFileSystem.PATH_SUFFIX_JSON, (emptyPathSuffix) ? "" : status.getPath().getName());
051        json.put(HttpFSFileSystem.TYPE_JSON, HttpFSFileSystem.FILE_TYPE.getType(status).toString());
052        json.put(HttpFSFileSystem.LENGTH_JSON, status.getLen());
053        json.put(HttpFSFileSystem.OWNER_JSON, status.getOwner());
054        json.put(HttpFSFileSystem.GROUP_JSON, status.getGroup());
055        json.put(HttpFSFileSystem.PERMISSION_JSON, HttpFSFileSystem.permissionToString(status.getPermission()));
056        json.put(HttpFSFileSystem.ACCESS_TIME_JSON, status.getAccessTime());
057        json.put(HttpFSFileSystem.MODIFICATION_TIME_JSON, status.getModificationTime());
058        json.put(HttpFSFileSystem.BLOCK_SIZE_JSON, status.getBlockSize());
059        json.put(HttpFSFileSystem.REPLICATION_JSON, status.getReplication());
060        return json;
061      }
062    
063      /**
064       * Converts a FileSystemAccess <code>FileStatus</code> object into a JSON
065       * object.
066       *
067       * @param status FileSystemAccess file status.
068       *
069       * @return The JSON representation of the file status.
070       */
071      @SuppressWarnings({"unchecked", "deprecation"})
072      private static Map fileStatusToJSON(FileStatus status) {
073        Map json = new LinkedHashMap();
074        json.put(HttpFSFileSystem.FILE_STATUS_JSON, fileStatusToJSONRaw(status, true));
075        return json;
076      }
077    
078      /**
079       * Converts a <code>FileChecksum</code> object into a JSON array
080       * object.
081       *
082       * @param checksum file checksum.
083       *
084       * @return The JSON representation of the file checksum.
085       */
086      @SuppressWarnings({"unchecked"})
087      private static Map fileChecksumToJSON(FileChecksum checksum) {
088        Map json = new LinkedHashMap();
089        json.put(HttpFSFileSystem.CHECKSUM_ALGORITHM_JSON, checksum.getAlgorithmName());
090        json.put(HttpFSFileSystem.CHECKSUM_BYTES_JSON,
091                 org.apache.hadoop.util.StringUtils.byteToHexString(checksum.getBytes()));
092        json.put(HttpFSFileSystem.CHECKSUM_LENGTH_JSON, checksum.getLength());
093        Map response = new LinkedHashMap();
094        response.put(HttpFSFileSystem.FILE_CHECKSUM_JSON, json);
095        return response;
096      }
097    
098      /**
099       * Converts a <code>ContentSummary</code> object into a JSON array
100       * object.
101       *
102       * @param contentSummary the content summary
103       *
104       * @return The JSON representation of the content summary.
105       */
106      @SuppressWarnings({"unchecked"})
107      private static Map contentSummaryToJSON(ContentSummary contentSummary) {
108        Map json = new LinkedHashMap();
109        json.put(HttpFSFileSystem.CONTENT_SUMMARY_DIRECTORY_COUNT_JSON, contentSummary.getDirectoryCount());
110        json.put(HttpFSFileSystem.CONTENT_SUMMARY_FILE_COUNT_JSON, contentSummary.getFileCount());
111        json.put(HttpFSFileSystem.CONTENT_SUMMARY_LENGTH_JSON, contentSummary.getLength());
112        json.put(HttpFSFileSystem.CONTENT_SUMMARY_QUOTA_JSON, contentSummary.getQuota());
113        json.put(HttpFSFileSystem.CONTENT_SUMMARY_SPACE_CONSUMED_JSON, contentSummary.getSpaceConsumed());
114        json.put(HttpFSFileSystem.CONTENT_SUMMARY_SPACE_QUOTA_JSON, contentSummary.getSpaceQuota());
115        Map response = new LinkedHashMap();
116        response.put(HttpFSFileSystem.CONTENT_SUMMARY_JSON, json);
117        return response;
118      }
119    
120      /**
121       * Converts a FileSystemAccess <code>FileStatus</code> array into a JSON array
122       * object.
123       *
124       * @param status FileSystemAccess file status array.
125       * <code>SCHEME://HOST:PORT</code> in the file status.
126       *
127       * @return The JSON representation of the file status array.
128       */
129      @SuppressWarnings("unchecked")
130      private static Map fileStatusToJSON(FileStatus[] status) {
131        JSONArray json = new JSONArray();
132        if (status != null) {
133          for (FileStatus s : status) {
134            json.add(fileStatusToJSONRaw(s, false));
135          }
136        }
137        Map response = new LinkedHashMap();
138        Map temp = new LinkedHashMap();
139        temp.put(HttpFSFileSystem.FILE_STATUS_JSON, json);
140        response.put(HttpFSFileSystem.FILE_STATUSES_JSON, temp);
141        return response;
142      }
143    
144      /**
145       * Converts an object into a Json Map with with one key-value entry.
146       * <p/>
147       * It assumes the given value is either a JSON primitive type or a
148       * <code>JsonAware</code> instance.
149       *
150       * @param name name for the key of the entry.
151       * @param value for the value of the entry.
152       *
153       * @return the JSON representation of the key-value pair.
154       */
155      @SuppressWarnings("unchecked")
156      private static JSONObject toJSON(String name, Object value) {
157        JSONObject json = new JSONObject();
158        json.put(name, value);
159        return json;
160      }
161    
162      /**
163       * Executor that performs an append FileSystemAccess files system operation.
164       */
165      @InterfaceAudience.Private
166      public static class FSAppend implements FileSystemAccess.FileSystemExecutor<Void> {
167        private InputStream is;
168        private Path path;
169    
170        /**
171         * Creates an Append executor.
172         *
173         * @param is input stream to append.
174         * @param path path of the file to append.
175         */
176        public FSAppend(InputStream is, String path) {
177          this.is = is;
178          this.path = new Path(path);
179        }
180    
181        /**
182         * Executes the filesystem operation.
183         *
184         * @param fs filesystem instance to use.
185         *
186         * @return void.
187         *
188         * @throws IOException thrown if an IO error occured.
189         */
190        @Override
191        public Void execute(FileSystem fs) throws IOException {
192          int bufferSize = fs.getConf().getInt("httpfs.buffer.size", 4096);
193          OutputStream os = fs.append(path, bufferSize);
194          IOUtils.copyBytes(is, os, bufferSize, true);
195          os.close();
196          return null;
197        }
198    
199      }
200    
201      /**
202       * Executor that performs a content-summary FileSystemAccess files system operation.
203       */
204      @InterfaceAudience.Private
205      public static class FSContentSummary implements FileSystemAccess.FileSystemExecutor<Map> {
206        private Path path;
207    
208        /**
209         * Creates a content-summary executor.
210         *
211         * @param path the path to retrieve the content-summary.
212         */
213        public FSContentSummary(String path) {
214          this.path = new Path(path);
215        }
216    
217        /**
218         * Executes the filesystem operation.
219         *
220         * @param fs filesystem instance to use.
221         *
222         * @return a Map object (JSON friendly) with the content-summary.
223         *
224         * @throws IOException thrown if an IO error occured.
225         */
226        @Override
227        public Map execute(FileSystem fs) throws IOException {
228          ContentSummary contentSummary = fs.getContentSummary(path);
229          return contentSummaryToJSON(contentSummary);
230        }
231    
232      }
233    
234      /**
235       * Executor that performs a create FileSystemAccess files system operation.
236       */
237      @InterfaceAudience.Private
238      public static class FSCreate implements FileSystemAccess.FileSystemExecutor<Void> {
239        private InputStream is;
240        private Path path;
241        private short permission;
242        private boolean override;
243        private short replication;
244        private long blockSize;
245    
246        /**
247         * Creates a Create executor.
248         *
249         * @param is input stream to for the file to create.
250         * @param path path of the file to create.
251         * @param perm permission for the file.
252         * @param override if the file should be overriden if it already exist.
253         * @param repl the replication factor for the file.
254         * @param blockSize the block size for the file.
255         */
256        public FSCreate(InputStream is, String path, short perm, boolean override,
257                        short repl, long blockSize) {
258          this.is = is;
259          this.path = new Path(path);
260          this.permission = perm;
261          this.override = override;
262          this.replication = repl;
263          this.blockSize = blockSize;
264        }
265    
266        /**
267         * Executes the filesystem operation.
268         *
269         * @param fs filesystem instance to use.
270         *
271         * @return The URI of the created file.
272         *
273         * @throws IOException thrown if an IO error occured.
274         */
275        @Override
276        public Void execute(FileSystem fs) throws IOException {
277          if (replication == -1) {
278            replication = fs.getDefaultReplication(path);
279          }
280          if (blockSize == -1) {
281            blockSize = fs.getDefaultBlockSize(path);
282          }
283          FsPermission fsPermission = new FsPermission(permission);
284          int bufferSize = fs.getConf().getInt("httpfs.buffer.size", 4096);
285          OutputStream os = fs.create(path, fsPermission, override, bufferSize, replication, blockSize, null);
286          IOUtils.copyBytes(is, os, bufferSize, true);
287          os.close();
288          return null;
289        }
290    
291      }
292    
293      /**
294       * Executor that performs a delete FileSystemAccess files system operation.
295       */
296      @InterfaceAudience.Private
297      public static class FSDelete implements FileSystemAccess.FileSystemExecutor<JSONObject> {
298        private Path path;
299        private boolean recursive;
300    
301        /**
302         * Creates a Delete executor.
303         *
304         * @param path path to delete.
305         * @param recursive if the delete should be recursive or not.
306         */
307        public FSDelete(String path, boolean recursive) {
308          this.path = new Path(path);
309          this.recursive = recursive;
310        }
311    
312        /**
313         * Executes the filesystem operation.
314         *
315         * @param fs filesystem instance to use.
316         *
317         * @return <code>true</code> if the delete operation was successful,
318         *         <code>false</code> otherwise.
319         *
320         * @throws IOException thrown if an IO error occured.
321         */
322        @Override
323        public JSONObject execute(FileSystem fs) throws IOException {
324          boolean deleted = fs.delete(path, recursive);
325          return toJSON(HttpFSFileSystem.DELETE_JSON.toLowerCase(), deleted);
326        }
327    
328      }
329    
330      /**
331       * Executor that performs a file-checksum FileSystemAccess files system operation.
332       */
333      @InterfaceAudience.Private
334      public static class FSFileChecksum implements FileSystemAccess.FileSystemExecutor<Map> {
335        private Path path;
336    
337        /**
338         * Creates a file-checksum executor.
339         *
340         * @param path the path to retrieve the checksum.
341         */
342        public FSFileChecksum(String path) {
343          this.path = new Path(path);
344        }
345    
346        /**
347         * Executes the filesystem operation.
348         *
349         * @param fs filesystem instance to use.
350         *
351         * @return a Map object (JSON friendly) with the file checksum.
352         *
353         * @throws IOException thrown if an IO error occured.
354         */
355        @Override
356        public Map execute(FileSystem fs) throws IOException {
357          FileChecksum checksum = fs.getFileChecksum(path);
358          return fileChecksumToJSON(checksum);
359        }
360    
361      }
362    
363      /**
364       * Executor that performs a file-status FileSystemAccess files system operation.
365       */
366      @InterfaceAudience.Private
367      public static class FSFileStatus implements FileSystemAccess.FileSystemExecutor<Map> {
368        private Path path;
369    
370        /**
371         * Creates a file-status executor.
372         *
373         * @param path the path to retrieve the status.
374         */
375        public FSFileStatus(String path) {
376          this.path = new Path(path);
377        }
378    
379        /**
380         * Executes the filesystem operation.
381         *
382         * @param fs filesystem instance to use.
383         *
384         * @return a Map object (JSON friendly) with the file status.
385         *
386         * @throws IOException thrown if an IO error occured.
387         */
388        @Override
389        public Map execute(FileSystem fs) throws IOException {
390          FileStatus status = fs.getFileStatus(path);
391          return fileStatusToJSON(status);
392        }
393    
394      }
395    
396      /**
397       * Executor that performs a home-dir FileSystemAccess files system operation.
398       */
399      @InterfaceAudience.Private
400      public static class FSHomeDir implements FileSystemAccess.FileSystemExecutor<JSONObject> {
401    
402        /**
403         * Executes the filesystem operation.
404         *
405         * @param fs filesystem instance to use.
406         *
407         * @return a JSON object with the user home directory.
408         *
409         * @throws IOException thrown if an IO error occured.
410         */
411        @Override
412        @SuppressWarnings("unchecked")
413        public JSONObject execute(FileSystem fs) throws IOException {
414          Path homeDir = fs.getHomeDirectory();
415          JSONObject json = new JSONObject();
416          json.put(HttpFSFileSystem.HOME_DIR_JSON, homeDir.toUri().getPath());
417          return json;
418        }
419    
420      }
421    
422      /**
423       * Executor that performs a list-status FileSystemAccess files system operation.
424       */
425      @InterfaceAudience.Private
426      public static class FSListStatus implements FileSystemAccess.FileSystemExecutor<Map>, PathFilter {
427        private Path path;
428        private PathFilter filter;
429    
430        /**
431         * Creates a list-status executor.
432         *
433         * @param path the directory to retrieve the status of its contents.
434         * @param filter glob filter to use.
435         *
436         * @throws IOException thrown if the filter expression is incorrect.
437         */
438        public FSListStatus(String path, String filter) throws IOException {
439          this.path = new Path(path);
440          this.filter = (filter == null) ? this : new GlobFilter(filter);
441        }
442    
443        /**
444         * Executes the filesystem operation.
445         *
446         * @param fs filesystem instance to use.
447         *
448         * @return a Map with the file status of the directory
449         *         contents.
450         *
451         * @throws IOException thrown if an IO error occured.
452         */
453        @Override
454        public Map execute(FileSystem fs) throws IOException {
455          FileStatus[] status = fs.listStatus(path, filter);
456          return fileStatusToJSON(status);
457        }
458    
459        @Override
460        public boolean accept(Path path) {
461          return true;
462        }
463    
464      }
465    
466      /**
467       * Executor that performs a mkdirs FileSystemAccess files system operation.
468       */
469      @InterfaceAudience.Private
470      public static class FSMkdirs implements FileSystemAccess.FileSystemExecutor<JSONObject> {
471    
472        private Path path;
473        private short permission;
474    
475        /**
476         * Creates a mkdirs executor.
477         *
478         * @param path directory path to create.
479         * @param permission permission to use.
480         */
481        public FSMkdirs(String path, short permission) {
482          this.path = new Path(path);
483          this.permission = permission;
484        }
485    
486        /**
487         * Executes the filesystem operation.
488         *
489         * @param fs filesystem instance to use.
490         *
491         * @return <code>true</code> if the mkdirs operation was successful,
492         *         <code>false</code> otherwise.
493         *
494         * @throws IOException thrown if an IO error occured.
495         */
496        @Override
497        public JSONObject execute(FileSystem fs) throws IOException {
498          FsPermission fsPermission = new FsPermission(permission);
499          boolean mkdirs = fs.mkdirs(path, fsPermission);
500          return toJSON(HttpFSFileSystem.MKDIRS_JSON, mkdirs);
501        }
502    
503      }
504    
505      /**
506       * Executor that performs a open FileSystemAccess files system operation.
507       */
508      @InterfaceAudience.Private
509      public static class FSOpen implements FileSystemAccess.FileSystemExecutor<InputStream> {
510        private Path path;
511    
512        /**
513         * Creates a open executor.
514         *
515         * @param path file to open.
516         */
517        public FSOpen(String path) {
518          this.path = new Path(path);
519        }
520    
521        /**
522         * Executes the filesystem operation.
523         *
524         * @param fs filesystem instance to use.
525         *
526         * @return The inputstream of the file.
527         *
528         * @throws IOException thrown if an IO error occured.
529         */
530        @Override
531        public InputStream execute(FileSystem fs) throws IOException {
532          int bufferSize = HttpFSServerWebApp.get().getConfig().getInt("httpfs.buffer.size", 4096);
533          return fs.open(path, bufferSize);
534        }
535    
536      }
537    
538      /**
539       * Executor that performs a rename FileSystemAccess files system operation.
540       */
541      @InterfaceAudience.Private
542      public static class FSRename implements FileSystemAccess.FileSystemExecutor<JSONObject> {
543        private Path path;
544        private Path toPath;
545    
546        /**
547         * Creates a rename executor.
548         *
549         * @param path path to rename.
550         * @param toPath new name.
551         */
552        public FSRename(String path, String toPath) {
553          this.path = new Path(path);
554          this.toPath = new Path(toPath);
555        }
556    
557        /**
558         * Executes the filesystem operation.
559         *
560         * @param fs filesystem instance to use.
561         *
562         * @return <code>true</code> if the rename operation was successful,
563         *         <code>false</code> otherwise.
564         *
565         * @throws IOException thrown if an IO error occured.
566         */
567        @Override
568        public JSONObject execute(FileSystem fs) throws IOException {
569          boolean renamed = fs.rename(path, toPath);
570          return toJSON(HttpFSFileSystem.RENAME_JSON, renamed);
571        }
572    
573      }
574    
575      /**
576       * Executor that performs a set-owner FileSystemAccess files system operation.
577       */
578      @InterfaceAudience.Private
579      public static class FSSetOwner implements FileSystemAccess.FileSystemExecutor<Void> {
580        private Path path;
581        private String owner;
582        private String group;
583    
584        /**
585         * Creates a set-owner executor.
586         *
587         * @param path the path to set the owner.
588         * @param owner owner to set.
589         * @param group group to set.
590         */
591        public FSSetOwner(String path, String owner, String group) {
592          this.path = new Path(path);
593          this.owner = owner;
594          this.group = group;
595        }
596    
597        /**
598         * Executes the filesystem operation.
599         *
600         * @param fs filesystem instance to use.
601         *
602         * @return void.
603         *
604         * @throws IOException thrown if an IO error occured.
605         */
606        @Override
607        public Void execute(FileSystem fs) throws IOException {
608          fs.setOwner(path, owner, group);
609          return null;
610        }
611    
612      }
613    
614      /**
615       * Executor that performs a set-permission FileSystemAccess files system operation.
616       */
617      @InterfaceAudience.Private
618      public static class FSSetPermission implements FileSystemAccess.FileSystemExecutor<Void> {
619    
620        private Path path;
621        private short permission;
622    
623        /**
624         * Creates a set-permission executor.
625         *
626         * @param path path to set the permission.
627         * @param permission permission to set.
628         */
629        public FSSetPermission(String path, short permission) {
630          this.path = new Path(path);
631          this.permission = permission;
632        }
633    
634        /**
635         * Executes the filesystem operation.
636         *
637         * @param fs filesystem instance to use.
638         *
639         * @return void.
640         *
641         * @throws IOException thrown if an IO error occured.
642         */
643        @Override
644        public Void execute(FileSystem fs) throws IOException {
645          FsPermission fsPermission = new FsPermission(permission);
646          fs.setPermission(path, fsPermission);
647          return null;
648        }
649    
650      }
651    
652      /**
653       * Executor that performs a set-replication FileSystemAccess files system operation.
654       */
655      @InterfaceAudience.Private
656      public static class FSSetReplication implements FileSystemAccess.FileSystemExecutor<JSONObject> {
657        private Path path;
658        private short replication;
659    
660        /**
661         * Creates a set-replication executor.
662         *
663         * @param path path to set the replication factor.
664         * @param replication replication factor to set.
665         */
666        public FSSetReplication(String path, short replication) {
667          this.path = new Path(path);
668          this.replication = replication;
669        }
670    
671        /**
672         * Executes the filesystem operation.
673         *
674         * @param fs filesystem instance to use.
675         *
676         * @return <code>true</code> if the replication value was set,
677         *         <code>false</code> otherwise.
678         *
679         * @throws IOException thrown if an IO error occured.
680         */
681        @Override
682        @SuppressWarnings("unchecked")
683        public JSONObject execute(FileSystem fs) throws IOException {
684          boolean ret = fs.setReplication(path, replication);
685          JSONObject json = new JSONObject();
686          json.put(HttpFSFileSystem.SET_REPLICATION_JSON, ret);
687          return json;
688        }
689    
690      }
691    
692      /**
693       * Executor that performs a set-times FileSystemAccess files system operation.
694       */
695      @InterfaceAudience.Private
696      public static class FSSetTimes implements FileSystemAccess.FileSystemExecutor<Void> {
697        private Path path;
698        private long mTime;
699        private long aTime;
700    
701        /**
702         * Creates a set-times executor.
703         *
704         * @param path path to set the times.
705         * @param mTime modified time to set.
706         * @param aTime access time to set.
707         */
708        public FSSetTimes(String path, long mTime, long aTime) {
709          this.path = new Path(path);
710          this.mTime = mTime;
711          this.aTime = aTime;
712        }
713    
714        /**
715         * Executes the filesystem operation.
716         *
717         * @param fs filesystem instance to use.
718         *
719         * @return void.
720         *
721         * @throws IOException thrown if an IO error occured.
722         */
723        @Override
724        public Void execute(FileSystem fs) throws IOException {
725          fs.setTimes(path, mTime, aTime);
726          return null;
727        }
728    
729      }
730    
731    }