001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.hadoop.fs;
019
020 import java.io.Closeable;
021 import java.io.FileNotFoundException;
022 import java.io.IOException;
023
024 import java.net.InetSocketAddress;
025 import java.net.URI;
026 import java.net.URISyntaxException;
027 import java.security.PrivilegedExceptionAction;
028 import java.util.ArrayList;
029 import java.util.Arrays;
030 import java.util.Collections;
031 import java.util.EnumSet;
032 import java.util.HashMap;
033 import java.util.HashSet;
034 import java.util.IdentityHashMap;
035 import java.util.Iterator;
036 import java.util.List;
037 import java.util.Map;
038 import java.util.NoSuchElementException;
039 import java.util.ServiceLoader;
040 import java.util.Set;
041 import java.util.Stack;
042 import java.util.TreeSet;
043 import java.util.concurrent.atomic.AtomicInteger;
044 import java.util.concurrent.atomic.AtomicLong;
045
046 import org.apache.commons.logging.Log;
047 import org.apache.commons.logging.LogFactory;
048 import org.apache.hadoop.classification.InterfaceAudience;
049 import org.apache.hadoop.classification.InterfaceStability;
050 import org.apache.hadoop.conf.Configuration;
051 import org.apache.hadoop.conf.Configured;
052 import org.apache.hadoop.fs.Options.ChecksumOpt;
053 import org.apache.hadoop.fs.Options.Rename;
054 import org.apache.hadoop.fs.permission.FsPermission;
055 import org.apache.hadoop.io.MultipleIOException;
056 import org.apache.hadoop.io.Text;
057 import org.apache.hadoop.net.NetUtils;
058 import org.apache.hadoop.security.AccessControlException;
059 import org.apache.hadoop.security.Credentials;
060 import org.apache.hadoop.security.SecurityUtil;
061 import org.apache.hadoop.security.UserGroupInformation;
062 import org.apache.hadoop.security.token.Token;
063 import org.apache.hadoop.util.DataChecksum;
064 import org.apache.hadoop.util.Progressable;
065 import org.apache.hadoop.util.ReflectionUtils;
066 import org.apache.hadoop.util.ShutdownHookManager;
067
068 import com.google.common.annotations.VisibleForTesting;
069
070 /****************************************************************
071 * An abstract base class for a fairly generic filesystem. It
072 * may be implemented as a distributed filesystem, or as a "local"
073 * one that reflects the locally-connected disk. The local version
074 * exists for small Hadoop instances and for testing.
075 *
076 * <p>
077 *
078 * All user code that may potentially use the Hadoop Distributed
079 * File System should be written to use a FileSystem object. The
080 * Hadoop DFS is a multi-machine system that appears as a single
081 * disk. It's useful because of its fault tolerance and potentially
082 * very large capacity.
083 *
084 * <p>
085 * The local implementation is {@link LocalFileSystem} and distributed
086 * implementation is DistributedFileSystem.
087 *****************************************************************/
088 @InterfaceAudience.Public
089 @InterfaceStability.Stable
090 public abstract class FileSystem extends Configured implements Closeable {
091 public static final String FS_DEFAULT_NAME_KEY =
092 CommonConfigurationKeys.FS_DEFAULT_NAME_KEY;
093 public static final String DEFAULT_FS =
094 CommonConfigurationKeys.FS_DEFAULT_NAME_DEFAULT;
095
096 public static final Log LOG = LogFactory.getLog(FileSystem.class);
097
098 /**
099 * Priority of the FileSystem shutdown hook.
100 */
101 public static final int SHUTDOWN_HOOK_PRIORITY = 10;
102
103 /** FileSystem cache */
104 static final Cache CACHE = new Cache();
105
106 /** The key this instance is stored under in the cache. */
107 private Cache.Key key;
108
109 /** Recording statistics per a FileSystem class */
110 private static final Map<Class<? extends FileSystem>, Statistics>
111 statisticsTable =
112 new IdentityHashMap<Class<? extends FileSystem>, Statistics>();
113
114 /**
115 * The statistics for this file system.
116 */
117 protected Statistics statistics;
118
119 /**
120 * A cache of files that should be deleted when filsystem is closed
121 * or the JVM is exited.
122 */
123 private Set<Path> deleteOnExit = new TreeSet<Path>();
124
125 boolean resolveSymlinks;
126 /**
127 * This method adds a file system for testing so that we can find it later. It
128 * is only for testing.
129 * @param uri the uri to store it under
130 * @param conf the configuration to store it under
131 * @param fs the file system to store
132 * @throws IOException
133 */
134 static void addFileSystemForTesting(URI uri, Configuration conf,
135 FileSystem fs) throws IOException {
136 CACHE.map.put(new Cache.Key(uri, conf), fs);
137 }
138
139 /**
140 * Get a filesystem instance based on the uri, the passed
141 * configuration and the user
142 * @param uri of the filesystem
143 * @param conf the configuration to use
144 * @param user to perform the get as
145 * @return the filesystem instance
146 * @throws IOException
147 * @throws InterruptedException
148 */
149 public static FileSystem get(final URI uri, final Configuration conf,
150 final String user) throws IOException, InterruptedException {
151 String ticketCachePath =
152 conf.get(CommonConfigurationKeys.KERBEROS_TICKET_CACHE_PATH);
153 UserGroupInformation ugi =
154 UserGroupInformation.getBestUGI(ticketCachePath, user);
155 return ugi.doAs(new PrivilegedExceptionAction<FileSystem>() {
156 @Override
157 public FileSystem run() throws IOException {
158 return get(uri, conf);
159 }
160 });
161 }
162
163 /**
164 * Returns the configured filesystem implementation.
165 * @param conf the configuration to use
166 */
167 public static FileSystem get(Configuration conf) throws IOException {
168 return get(getDefaultUri(conf), conf);
169 }
170
171 /** Get the default filesystem URI from a configuration.
172 * @param conf the configuration to use
173 * @return the uri of the default filesystem
174 */
175 public static URI getDefaultUri(Configuration conf) {
176 return URI.create(fixName(conf.get(FS_DEFAULT_NAME_KEY, DEFAULT_FS)));
177 }
178
179 /**
180 * create PathId: A Factory method to create PathIds. PathIds are used
181 * by MapRFS in the direct shuffle to get access to the file paths.
182 */
183 public PathId createPathId() {
184 //Base class throws Unsupported Exception
185 //TODO determine if this is the right approach
186 throw new UnsupportedOperationException();
187 }
188
189 public FSDataInputStream openFid2(PathId pfid, String file, int readAheadBytesHint)
190 throws IOException {
191 throw new UnsupportedOperationException("See concrete FS for implementation");
192 }
193 public FSDataOutputStream createFid(String pfid, String file)
194 throws IOException {
195 throw new UnsupportedOperationException("See concrete FS for implementation");
196 }
197 public boolean deleteFid(String pfid, String dir)
198 throws IOException {
199 throw new UnsupportedOperationException("See concrete FS for implementation");
200 }
201 public String mkdirsFid(Path p) throws IOException {
202 throw new UnsupportedOperationException("See concrete FS for implementation");
203 }
204 public String mkdirsFid(String pfid, String dir)
205 throws IOException {
206 throw new UnsupportedOperationException("See concrete FS for implementation");
207 }
208 public void setOwnerFid(String fid, String user, String group) throws IOException {
209 throw new UnsupportedOperationException("See concrete FS for implementation");
210 }
211
212
213 /** Set the default filesystem URI in a configuration.
214 * @param conf the configuration to alter
215 * @param uri the new default filesystem uri
216 */
217 public static void setDefaultUri(Configuration conf, URI uri) {
218 conf.set(FS_DEFAULT_NAME_KEY, uri.toString());
219 }
220
221 /** Set the default filesystem URI in a configuration.
222 * @param conf the configuration to alter
223 * @param uri the new default filesystem uri
224 */
225 public static void setDefaultUri(Configuration conf, String uri) {
226 setDefaultUri(conf, URI.create(fixName(uri)));
227 }
228
229 /** Called after a new FileSystem instance is constructed.
230 * @param name a uri whose authority section names the host, port, etc.
231 * for this FileSystem
232 * @param conf the configuration
233 */
234 public void initialize(URI name, Configuration conf) throws IOException {
235 statistics = getStatistics(name.getScheme(), getClass());
236 resolveSymlinks = conf.getBoolean(
237 CommonConfigurationKeys.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY,
238 CommonConfigurationKeys.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_DEFAULT);
239 }
240
241 /**
242 * Return the protocol scheme for the FileSystem.
243 * <p/>
244 * This implementation throws an <code>UnsupportedOperationException</code>.
245 *
246 * @return the protocol scheme for the FileSystem.
247 */
248 public String getScheme() {
249 throw new UnsupportedOperationException("Not implemented by the " + getClass().getSimpleName() + " FileSystem implementation");
250 }
251
252 /** Returns a URI whose scheme and authority identify this FileSystem.*/
253 public abstract URI getUri();
254
255 /**
256 * Return a canonicalized form of this FileSystem's URI.
257 *
258 * The default implementation simply calls {@link #canonicalizeUri(URI)}
259 * on the filesystem's own URI, so subclasses typically only need to
260 * implement that method.
261 *
262 * @see #canonicalizeUri(URI)
263 */
264 protected URI getCanonicalUri() {
265 return canonicalizeUri(getUri());
266 }
267
268 /**
269 * Canonicalize the given URI.
270 *
271 * This is filesystem-dependent, but may for example consist of
272 * canonicalizing the hostname using DNS and adding the default
273 * port if not specified.
274 *
275 * The default implementation simply fills in the default port if
276 * not specified and if the filesystem has a default port.
277 *
278 * @return URI
279 * @see NetUtils#getCanonicalUri(URI, int)
280 */
281 protected URI canonicalizeUri(URI uri) {
282 if (uri.getPort() == -1 && getDefaultPort() > 0) {
283 // reconstruct the uri with the default port set
284 try {
285 uri = new URI(uri.getScheme(), uri.getUserInfo(),
286 uri.getHost(), getDefaultPort(),
287 uri.getPath(), uri.getQuery(), uri.getFragment());
288 } catch (URISyntaxException e) {
289 // Should never happen!
290 throw new AssertionError("Valid URI became unparseable: " +
291 uri);
292 }
293 }
294
295 return uri;
296 }
297
298 /**
299 * Get the default port for this file system.
300 * @return the default port or 0 if there isn't one
301 */
302 protected int getDefaultPort() {
303 return 0;
304 }
305
306 protected static FileSystem getFSofPath(final Path absOrFqPath,
307 final Configuration conf)
308 throws UnsupportedFileSystemException, IOException {
309 absOrFqPath.checkNotSchemeWithRelative();
310 absOrFqPath.checkNotRelative();
311
312 // Uses the default file system if not fully qualified
313 return get(absOrFqPath.toUri(), conf);
314 }
315
316 /**
317 * Get a canonical service name for this file system. The token cache is
318 * the only user of the canonical service name, and uses it to lookup this
319 * filesystem's service tokens.
320 * If file system provides a token of its own then it must have a canonical
321 * name, otherwise canonical name can be null.
322 *
323 * Default Impl: If the file system has child file systems
324 * (such as an embedded file system) then it is assumed that the fs has no
325 * tokens of its own and hence returns a null name; otherwise a service
326 * name is built using Uri and port.
327 *
328 * @return a service string that uniquely identifies this file system, null
329 * if the filesystem does not implement tokens
330 * @see SecurityUtil#buildDTServiceName(URI, int)
331 */
332 @InterfaceAudience.LimitedPrivate({ "HDFS", "MapReduce" })
333 public String getCanonicalServiceName() {
334 return (getChildFileSystems() == null)
335 ? SecurityUtil.buildDTServiceName(getUri(), getDefaultPort())
336 : null;
337 }
338
339 /** @deprecated call #getUri() instead.*/
340 @Deprecated
341 public String getName() { return getUri().toString(); }
342
343 /** @deprecated call #get(URI,Configuration) instead. */
344 @Deprecated
345 public static FileSystem getNamed(String name, Configuration conf)
346 throws IOException {
347 return get(URI.create(fixName(name)), conf);
348 }
349
350 /** Update old-format filesystem names, for back-compatibility. This should
351 * eventually be replaced with a checkName() method that throws an exception
352 * for old-format names. */
353 private static String fixName(String name) {
354 // convert old-format name to new-format name
355 if (name.equals("local")) { // "local" is now "file:///".
356 LOG.warn("\"local\" is a deprecated filesystem name."
357 +" Use \"file:///\" instead.");
358 name = "file:///";
359 } else if (name.indexOf('/')==-1) { // unqualified is "hdfs://"
360 LOG.warn("\""+name+"\" is a deprecated filesystem name."
361 +" Use \"hdfs://"+name+"/\" instead.");
362 name = "hdfs://"+name;
363 }
364 return name;
365 }
366
367 /**
368 * Get the local file system.
369 * @param conf the configuration to configure the file system with
370 * @return a LocalFileSystem
371 */
372 public static LocalFileSystem getLocal(Configuration conf)
373 throws IOException {
374 return (LocalFileSystem)get(LocalFileSystem.NAME, conf);
375 }
376
377 /** Returns the FileSystem for this URI's scheme and authority. The scheme
378 * of the URI determines a configuration property name,
379 * <tt>fs.<i>scheme</i>.class</tt> whose value names the FileSystem class.
380 * The entire URI is passed to the FileSystem instance's initialize method.
381 */
382 public static FileSystem get(URI uri, Configuration conf) throws IOException {
383 String scheme = uri.getScheme();
384 String authority = uri.getAuthority();
385
386 if (scheme == null && authority == null) { // use default FS
387 return get(conf);
388 }
389
390 if (scheme != null && authority == null) { // no authority
391 URI defaultUri = getDefaultUri(conf);
392 if (scheme.equals(defaultUri.getScheme()) // if scheme matches default
393 && defaultUri.getAuthority() != null) { // & default has authority
394 return get(defaultUri, conf); // return default
395 }
396 }
397
398 String disableCacheName = String.format("fs.%s.impl.disable.cache", scheme);
399 if (conf.getBoolean(disableCacheName, false)) {
400 return createFileSystem(uri, conf);
401 }
402
403 return CACHE.get(uri, conf);
404 }
405
406 /**
407 * Returns the FileSystem for this URI's scheme and authority and the
408 * passed user. Internally invokes {@link #newInstance(URI, Configuration)}
409 * @param uri of the filesystem
410 * @param conf the configuration to use
411 * @param user to perform the get as
412 * @return filesystem instance
413 * @throws IOException
414 * @throws InterruptedException
415 */
416 public static FileSystem newInstance(final URI uri, final Configuration conf,
417 final String user) throws IOException, InterruptedException {
418 String ticketCachePath =
419 conf.get(CommonConfigurationKeys.KERBEROS_TICKET_CACHE_PATH);
420 UserGroupInformation ugi =
421 UserGroupInformation.getBestUGI(ticketCachePath, user);
422 return ugi.doAs(new PrivilegedExceptionAction<FileSystem>() {
423 @Override
424 public FileSystem run() throws IOException {
425 return newInstance(uri,conf);
426 }
427 });
428 }
429 /** Returns the FileSystem for this URI's scheme and authority. The scheme
430 * of the URI determines a configuration property name,
431 * <tt>fs.<i>scheme</i>.class</tt> whose value names the FileSystem class.
432 * The entire URI is passed to the FileSystem instance's initialize method.
433 * This always returns a new FileSystem object.
434 */
435 public static FileSystem newInstance(URI uri, Configuration conf) throws IOException {
436 String scheme = uri.getScheme();
437 String authority = uri.getAuthority();
438
439 if (scheme == null) { // no scheme: use default FS
440 return newInstance(conf);
441 }
442
443 if (authority == null) { // no authority
444 URI defaultUri = getDefaultUri(conf);
445 if (scheme.equals(defaultUri.getScheme()) // if scheme matches default
446 && defaultUri.getAuthority() != null) { // & default has authority
447 return newInstance(defaultUri, conf); // return default
448 }
449 }
450 return CACHE.getUnique(uri, conf);
451 }
452
453 /** Returns a unique configured filesystem implementation.
454 * This always returns a new FileSystem object.
455 * @param conf the configuration to use
456 */
457 public static FileSystem newInstance(Configuration conf) throws IOException {
458 return newInstance(getDefaultUri(conf), conf);
459 }
460
461 /**
462 * Get a unique local file system object
463 * @param conf the configuration to configure the file system with
464 * @return a LocalFileSystem
465 * This always returns a new FileSystem object.
466 */
467 public static LocalFileSystem newInstanceLocal(Configuration conf)
468 throws IOException {
469 return (LocalFileSystem)newInstance(LocalFileSystem.NAME, conf);
470 }
471
472 /**
473 * Close all cached filesystems. Be sure those filesystems are not
474 * used anymore.
475 *
476 * @throws IOException
477 */
478 public static void closeAll() throws IOException {
479 CACHE.closeAll();
480 }
481
482 /**
483 * Close all cached filesystems for a given UGI. Be sure those filesystems
484 * are not used anymore.
485 * @param ugi user group info to close
486 * @throws IOException
487 */
488 public static void closeAllForUGI(UserGroupInformation ugi)
489 throws IOException {
490 CACHE.closeAll(ugi);
491 }
492
493 /**
494 * Make sure that a path specifies a FileSystem.
495 * @param path to use
496 */
497 public Path makeQualified(Path path) {
498 checkPath(path);
499 return path.makeQualified(this.getUri(), this.getWorkingDirectory());
500 }
501
502 /**
503 * Get a new delegation token for this file system.
504 * This is an internal method that should have been declared protected
505 * but wasn't historically.
506 * Callers should use {@link #addDelegationTokens(String, Credentials)}
507 *
508 * @param renewer the account name that is allowed to renew the token.
509 * @return a new delegation token
510 * @throws IOException
511 */
512 @InterfaceAudience.Private()
513 public Token<?> getDelegationToken(String renewer) throws IOException {
514 return null;
515 }
516
517 /**
518 * Obtain all delegation tokens used by this FileSystem that are not
519 * already present in the given Credentials. Existing tokens will neither
520 * be verified as valid nor having the given renewer. Missing tokens will
521 * be acquired and added to the given Credentials.
522 *
523 * Default Impl: works for simple fs with its own token
524 * and also for an embedded fs whose tokens are those of its
525 * children file system (i.e. the embedded fs has not tokens of its
526 * own).
527 *
528 * @param renewer the user allowed to renew the delegation tokens
529 * @param credentials cache in which to add new delegation tokens
530 * @return list of new delegation tokens
531 * @throws IOException
532 */
533 @InterfaceAudience.LimitedPrivate({ "HDFS", "MapReduce" })
534 public Token<?>[] addDelegationTokens(
535 final String renewer, Credentials credentials) throws IOException {
536 if (credentials == null) {
537 credentials = new Credentials();
538 }
539 final List<Token<?>> tokens = new ArrayList<Token<?>>();
540 collectDelegationTokens(renewer, credentials, tokens);
541 return tokens.toArray(new Token<?>[tokens.size()]);
542 }
543
544 /**
545 * Recursively obtain the tokens for this FileSystem and all descended
546 * FileSystems as determined by getChildFileSystems().
547 * @param renewer the user allowed to renew the delegation tokens
548 * @param credentials cache in which to add the new delegation tokens
549 * @param tokens list in which to add acquired tokens
550 * @throws IOException
551 */
552 private void collectDelegationTokens(final String renewer,
553 final Credentials credentials,
554 final List<Token<?>> tokens)
555 throws IOException {
556 final String serviceName = getCanonicalServiceName();
557 // Collect token of the this filesystem and then of its embedded children
558 if (serviceName != null) { // fs has token, grab it
559 final Text service = new Text(serviceName);
560 Token<?> token = credentials.getToken(service);
561 if (token == null) {
562 token = getDelegationToken(renewer);
563 if (token != null) {
564 tokens.add(token);
565 credentials.addToken(service, token);
566 }
567 }
568 }
569 // Now collect the tokens from the children
570 final FileSystem[] children = getChildFileSystems();
571 if (children != null) {
572 for (final FileSystem fs : children) {
573 fs.collectDelegationTokens(renewer, credentials, tokens);
574 }
575 }
576 }
577
578 /**
579 * Get all the immediate child FileSystems embedded in this FileSystem.
580 * It does not recurse and get grand children. If a FileSystem
581 * has multiple child FileSystems, then it should return a unique list
582 * of those FileSystems. Default is to return null to signify no children.
583 *
584 * @return FileSystems used by this FileSystem
585 */
586 @InterfaceAudience.LimitedPrivate({ "HDFS" })
587 @VisibleForTesting
588 public FileSystem[] getChildFileSystems() {
589 return null;
590 }
591
592 /** create a file with the provided permission
593 * The permission of the file is set to be the provided permission as in
594 * setPermission, not permission&~umask
595 *
596 * It is implemented using two RPCs. It is understood that it is inefficient,
597 * but the implementation is thread-safe. The other option is to change the
598 * value of umask in configuration to be 0, but it is not thread-safe.
599 *
600 * @param fs file system handle
601 * @param file the name of the file to be created
602 * @param permission the permission of the file
603 * @return an output stream
604 * @throws IOException
605 */
606 public static FSDataOutputStream create(FileSystem fs,
607 Path file, FsPermission permission) throws IOException {
608 // create the file with default permission
609 FSDataOutputStream out = fs.create(file);
610 // set its permission to the supplied one
611 fs.setPermission(file, permission);
612 return out;
613 }
614
615 /** create a directory with the provided permission
616 * The permission of the directory is set to be the provided permission as in
617 * setPermission, not permission&~umask
618 *
619 * @see #create(FileSystem, Path, FsPermission)
620 *
621 * @param fs file system handle
622 * @param dir the name of the directory to be created
623 * @param permission the permission of the directory
624 * @return true if the directory creation succeeds; false otherwise
625 * @throws IOException
626 */
627 public static boolean mkdirs(FileSystem fs, Path dir, FsPermission permission)
628 throws IOException {
629 // create the directory using the default permission
630 boolean result = fs.mkdirs(dir);
631 // set its permission to be the supplied one
632 fs.setPermission(dir, permission);
633 return result;
634 }
635
636 ///////////////////////////////////////////////////////////////
637 // FileSystem
638 ///////////////////////////////////////////////////////////////
639
640 protected FileSystem() {
641 super(null);
642 }
643
644 /**
645 * Check that a Path belongs to this FileSystem.
646 * @param path to check
647 */
648 protected void checkPath(Path path) {
649 URI uri = path.toUri();
650 String thatScheme = uri.getScheme();
651 if (thatScheme == null) // fs is relative
652 return;
653 URI thisUri = getCanonicalUri();
654 String thisScheme = thisUri.getScheme();
655 //authority and scheme are not case sensitive
656 if (thisScheme.equalsIgnoreCase(thatScheme)) {// schemes match
657 String thisAuthority = thisUri.getAuthority();
658 String thatAuthority = uri.getAuthority();
659 if (thatAuthority == null && // path's authority is null
660 thisAuthority != null) { // fs has an authority
661 URI defaultUri = getDefaultUri(getConf());
662 if (thisScheme.equalsIgnoreCase(defaultUri.getScheme())) {
663 uri = defaultUri; // schemes match, so use this uri instead
664 } else {
665 uri = null; // can't determine auth of the path
666 }
667 }
668 if (uri != null) {
669 // canonicalize uri before comparing with this fs
670 uri = canonicalizeUri(uri);
671 thatAuthority = uri.getAuthority();
672 if (thisAuthority == thatAuthority || // authorities match
673 (thisAuthority != null &&
674 thisAuthority.equalsIgnoreCase(thatAuthority)))
675 return;
676 }
677 }
678 throw new IllegalArgumentException("Wrong FS: "+path+
679 ", expected: "+this.getUri());
680 }
681
682 /**
683 * Return an array containing hostnames, offset and size of
684 * portions of the given file. For a nonexistent
685 * file or regions, null will be returned.
686 *
687 * This call is most helpful with DFS, where it returns
688 * hostnames of machines that contain the given file.
689 *
690 * The FileSystem will simply return an elt containing 'localhost'.
691 *
692 * @param file FilesStatus to get data from
693 * @param start offset into the given file
694 * @param len length for which to get locations for
695 */
696 public BlockLocation[] getFileBlockLocations(FileStatus file,
697 long start, long len) throws IOException {
698 if (file == null) {
699 return null;
700 }
701
702 if (start < 0 || len < 0) {
703 throw new IllegalArgumentException("Invalid start or len parameter");
704 }
705
706 if (file.getLen() <= start) {
707 return new BlockLocation[0];
708
709 }
710 String[] name = { "localhost:50010" };
711 String[] host = { "localhost" };
712 return new BlockLocation[] {
713 new BlockLocation(name, host, 0, file.getLen()) };
714 }
715
716
717 /**
718 * Return an array containing hostnames, offset and size of
719 * portions of the given file. For a nonexistent
720 * file or regions, null will be returned.
721 *
722 * This call is most helpful with DFS, where it returns
723 * hostnames of machines that contain the given file.
724 *
725 * The FileSystem will simply return an elt containing 'localhost'.
726 *
727 * @param p path is used to identify an FS since an FS could have
728 * another FS that it could be delegating the call to
729 * @param start offset into the given file
730 * @param len length for which to get locations for
731 */
732 public BlockLocation[] getFileBlockLocations(Path p,
733 long start, long len) throws IOException {
734 if (p == null) {
735 throw new NullPointerException();
736 }
737 FileStatus file = getFileStatus(p);
738 return getFileBlockLocations(file, start, len);
739 }
740
741 /**
742 * Return a set of server default configuration values
743 * @return server default configuration values
744 * @throws IOException
745 * @deprecated use {@link #getServerDefaults(Path)} instead
746 */
747 @Deprecated
748 public FsServerDefaults getServerDefaults() throws IOException {
749 Configuration conf = getConf();
750 // CRC32 is chosen as default as it is available in all
751 // releases that support checksum.
752 // The client trash configuration is ignored.
753 return new FsServerDefaults(getDefaultBlockSize(),
754 conf.getInt("io.bytes.per.checksum", 512),
755 64 * 1024,
756 getDefaultReplication(),
757 conf.getInt("io.file.buffer.size", 4096),
758 false,
759 CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_DEFAULT,
760 DataChecksum.Type.CRC32);
761 }
762
763 /**
764 * Return a set of server default configuration values
765 * @param p path is used to identify an FS since an FS could have
766 * another FS that it could be delegating the call to
767 * @return server default configuration values
768 * @throws IOException
769 */
770 public FsServerDefaults getServerDefaults(Path p) throws IOException {
771 return getServerDefaults();
772 }
773
774 /**
775 * Return the fully-qualified path of path f resolving the path
776 * through any symlinks or mount point
777 * @param p path to be resolved
778 * @return fully qualified path
779 * @throws FileNotFoundException
780 */
781 public Path resolvePath(final Path p) throws IOException {
782 checkPath(p);
783 return getFileStatus(p).getPath();
784 }
785
786 /**
787 * Opens an FSDataInputStream at the indicated Path.
788 * @param f the file name to open
789 * @param bufferSize the size of the buffer to be used.
790 */
791 public abstract FSDataInputStream open(Path f, int bufferSize)
792 throws IOException;
793
794 /**
795 * Opens an FSDataInputStream at the indicated Path.
796 * @param f the file to open
797 */
798 public FSDataInputStream open(Path f) throws IOException {
799 return open(f, getConf().getInt("io.file.buffer.size", 4096));
800 }
801
802 /**
803 * Create an FSDataOutputStream at the indicated Path.
804 * Files are overwritten by default.
805 * @param f the file to create
806 */
807 public FSDataOutputStream create(Path f) throws IOException {
808 return create(f, true);
809 }
810
811 /**
812 * Create an FSDataOutputStream at the indicated Path.
813 * @param f the file to create
814 * @param overwrite if a file with this name already exists, then if true,
815 * the file will be overwritten, and if false an exception will be thrown.
816 */
817 public FSDataOutputStream create(Path f, boolean overwrite)
818 throws IOException {
819 return create(f, overwrite,
820 getConf().getInt("io.file.buffer.size", 4096),
821 getDefaultReplication(f),
822 getDefaultBlockSize(f));
823 }
824
825 /**
826 * Create an FSDataOutputStream at the indicated Path with write-progress
827 * reporting.
828 * Files are overwritten by default.
829 * @param f the file to create
830 * @param progress to report progress
831 */
832 public FSDataOutputStream create(Path f, Progressable progress)
833 throws IOException {
834 return create(f, true,
835 getConf().getInt("io.file.buffer.size", 4096),
836 getDefaultReplication(f),
837 getDefaultBlockSize(f), progress);
838 }
839
840 /**
841 * Create an FSDataOutputStream at the indicated Path.
842 * Files are overwritten by default.
843 * @param f the file to create
844 * @param replication the replication factor
845 */
846 public FSDataOutputStream create(Path f, short replication)
847 throws IOException {
848 return create(f, true,
849 getConf().getInt("io.file.buffer.size", 4096),
850 replication,
851 getDefaultBlockSize(f));
852 }
853
854 /**
855 * Create an FSDataOutputStream at the indicated Path with write-progress
856 * reporting.
857 * Files are overwritten by default.
858 * @param f the file to create
859 * @param replication the replication factor
860 * @param progress to report progress
861 */
862 public FSDataOutputStream create(Path f, short replication,
863 Progressable progress) throws IOException {
864 return create(f, true,
865 getConf().getInt(
866 CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY,
867 CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT),
868 replication,
869 getDefaultBlockSize(f), progress);
870 }
871
872
873 /**
874 * Create an FSDataOutputStream at the indicated Path.
875 * @param f the file name to create
876 * @param overwrite if a file with this name already exists, then if true,
877 * the file will be overwritten, and if false an error will be thrown.
878 * @param bufferSize the size of the buffer to be used.
879 */
880 public FSDataOutputStream create(Path f,
881 boolean overwrite,
882 int bufferSize
883 ) throws IOException {
884 return create(f, overwrite, bufferSize,
885 getDefaultReplication(f),
886 getDefaultBlockSize(f));
887 }
888
889 /**
890 * Create an FSDataOutputStream at the indicated Path with write-progress
891 * reporting.
892 * @param f the path of the file to open
893 * @param overwrite if a file with this name already exists, then if true,
894 * the file will be overwritten, and if false an error will be thrown.
895 * @param bufferSize the size of the buffer to be used.
896 */
897 public FSDataOutputStream create(Path f,
898 boolean overwrite,
899 int bufferSize,
900 Progressable progress
901 ) throws IOException {
902 return create(f, overwrite, bufferSize,
903 getDefaultReplication(f),
904 getDefaultBlockSize(f), progress);
905 }
906
907
908 /**
909 * Create an FSDataOutputStream at the indicated Path.
910 * @param f the file name to open
911 * @param overwrite if a file with this name already exists, then if true,
912 * the file will be overwritten, and if false an error will be thrown.
913 * @param bufferSize the size of the buffer to be used.
914 * @param replication required block replication for the file.
915 */
916 public FSDataOutputStream create(Path f,
917 boolean overwrite,
918 int bufferSize,
919 short replication,
920 long blockSize
921 ) throws IOException {
922 return create(f, overwrite, bufferSize, replication, blockSize, null);
923 }
924
925 /**
926 * Create an FSDataOutputStream at the indicated Path with write-progress
927 * reporting.
928 * @param f the file name to open
929 * @param overwrite if a file with this name already exists, then if true,
930 * the file will be overwritten, and if false an error will be thrown.
931 * @param bufferSize the size of the buffer to be used.
932 * @param replication required block replication for the file.
933 */
934 public FSDataOutputStream create(Path f,
935 boolean overwrite,
936 int bufferSize,
937 short replication,
938 long blockSize,
939 Progressable progress
940 ) throws IOException {
941 return this.create(f, FsPermission.getFileDefault().applyUMask(
942 FsPermission.getUMask(getConf())), overwrite, bufferSize,
943 replication, blockSize, progress);
944 }
945
946 /**
947 * Create an FSDataOutputStream at the indicated Path with write-progress
948 * reporting.
949 * @param f the file name to open
950 * @param permission
951 * @param overwrite if a file with this name already exists, then if true,
952 * the file will be overwritten, and if false an error will be thrown.
953 * @param bufferSize the size of the buffer to be used.
954 * @param replication required block replication for the file.
955 * @param blockSize
956 * @param progress
957 * @throws IOException
958 * @see #setPermission(Path, FsPermission)
959 */
960 public abstract FSDataOutputStream create(Path f,
961 FsPermission permission,
962 boolean overwrite,
963 int bufferSize,
964 short replication,
965 long blockSize,
966 Progressable progress) throws IOException;
967
968 /**
969 * Create an FSDataOutputStream at the indicated Path with write-progress
970 * reporting.
971 * @param f the file name to open
972 * @param permission
973 * @param flags {@link CreateFlag}s to use for this stream.
974 * @param bufferSize the size of the buffer to be used.
975 * @param replication required block replication for the file.
976 * @param blockSize
977 * @param progress
978 * @throws IOException
979 * @see #setPermission(Path, FsPermission)
980 */
981 public FSDataOutputStream create(Path f,
982 FsPermission permission,
983 EnumSet<CreateFlag> flags,
984 int bufferSize,
985 short replication,
986 long blockSize,
987 Progressable progress) throws IOException {
988 return create(f, permission, flags, bufferSize, replication,
989 blockSize, progress, null);
990 }
991
992 /**
993 * Create an FSDataOutputStream at the indicated Path with a custom
994 * checksum option
995 * @param f the file name to open
996 * @param permission
997 * @param flags {@link CreateFlag}s to use for this stream.
998 * @param bufferSize the size of the buffer to be used.
999 * @param replication required block replication for the file.
1000 * @param blockSize
1001 * @param progress
1002 * @param checksumOpt checksum parameter. If null, the values
1003 * found in conf will be used.
1004 * @throws IOException
1005 * @see #setPermission(Path, FsPermission)
1006 */
1007 public FSDataOutputStream create(Path f,
1008 FsPermission permission,
1009 EnumSet<CreateFlag> flags,
1010 int bufferSize,
1011 short replication,
1012 long blockSize,
1013 Progressable progress,
1014 ChecksumOpt checksumOpt) throws IOException {
1015 // Checksum options are ignored by default. The file systems that
1016 // implement checksum need to override this method. The full
1017 // support is currently only available in DFS.
1018 return create(f, permission, flags.contains(CreateFlag.OVERWRITE),
1019 bufferSize, replication, blockSize, progress);
1020 }
1021
1022 /*.
1023 * This create has been added to support the FileContext that processes
1024 * the permission
1025 * with umask before calling this method.
1026 * This a temporary method added to support the transition from FileSystem
1027 * to FileContext for user applications.
1028 */
1029 @Deprecated
1030 protected FSDataOutputStream primitiveCreate(Path f,
1031 FsPermission absolutePermission, EnumSet<CreateFlag> flag, int bufferSize,
1032 short replication, long blockSize, Progressable progress,
1033 ChecksumOpt checksumOpt) throws IOException {
1034
1035 boolean pathExists = exists(f);
1036 CreateFlag.validate(f, pathExists, flag);
1037
1038 // Default impl assumes that permissions do not matter and
1039 // nor does the bytesPerChecksum hence
1040 // calling the regular create is good enough.
1041 // FSs that implement permissions should override this.
1042
1043 if (pathExists && flag.contains(CreateFlag.APPEND)) {
1044 return append(f, bufferSize, progress);
1045 }
1046
1047 return this.create(f, absolutePermission,
1048 flag.contains(CreateFlag.OVERWRITE), bufferSize, replication,
1049 blockSize, progress);
1050 }
1051
1052 /**
1053 * This version of the mkdirs method assumes that the permission is absolute.
1054 * It has been added to support the FileContext that processes the permission
1055 * with umask before calling this method.
1056 * This a temporary method added to support the transition from FileSystem
1057 * to FileContext for user applications.
1058 */
1059 @Deprecated
1060 protected boolean primitiveMkdir(Path f, FsPermission absolutePermission)
1061 throws IOException {
1062 // Default impl is to assume that permissions do not matter and hence
1063 // calling the regular mkdirs is good enough.
1064 // FSs that implement permissions should override this.
1065 return this.mkdirs(f, absolutePermission);
1066 }
1067
1068
1069 /**
1070 * This version of the mkdirs method assumes that the permission is absolute.
1071 * It has been added to support the FileContext that processes the permission
1072 * with umask before calling this method.
1073 * This a temporary method added to support the transition from FileSystem
1074 * to FileContext for user applications.
1075 */
1076 @Deprecated
1077 protected void primitiveMkdir(Path f, FsPermission absolutePermission,
1078 boolean createParent)
1079 throws IOException {
1080
1081 if (!createParent) { // parent must exist.
1082 // since the this.mkdirs makes parent dirs automatically
1083 // we must throw exception if parent does not exist.
1084 final FileStatus stat = getFileStatus(f.getParent());
1085 if (stat == null) {
1086 throw new FileNotFoundException("Missing parent:" + f);
1087 }
1088 if (!stat.isDirectory()) {
1089 throw new ParentNotDirectoryException("parent is not a dir");
1090 }
1091 // parent does exist - go ahead with mkdir of leaf
1092 }
1093 // Default impl is to assume that permissions do not matter and hence
1094 // calling the regular mkdirs is good enough.
1095 // FSs that implement permissions should override this.
1096 if (!this.mkdirs(f, absolutePermission)) {
1097 throw new IOException("mkdir of "+ f + " failed");
1098 }
1099 }
1100
1101 /**
1102 * Opens an FSDataOutputStream at the indicated Path with write-progress
1103 * reporting. Same as create(), except fails if parent directory doesn't
1104 * already exist.
1105 * @param f the file name to open
1106 * @param overwrite if a file with this name already exists, then if true,
1107 * the file will be overwritten, and if false an error will be thrown.
1108 * @param bufferSize the size of the buffer to be used.
1109 * @param replication required block replication for the file.
1110 * @param blockSize
1111 * @param progress
1112 * @throws IOException
1113 * @see #setPermission(Path, FsPermission)
1114 * @deprecated API only for 0.20-append
1115 */
1116 @Deprecated
1117 public FSDataOutputStream createNonRecursive(Path f,
1118 boolean overwrite,
1119 int bufferSize, short replication, long blockSize,
1120 Progressable progress) throws IOException {
1121 return this.createNonRecursive(f, FsPermission.getFileDefault(),
1122 overwrite, bufferSize, replication, blockSize, progress);
1123 }
1124
1125 /**
1126 * Opens an FSDataOutputStream at the indicated Path with write-progress
1127 * reporting. Same as create(), except fails if parent directory doesn't
1128 * already exist.
1129 * @param f the file name to open
1130 * @param permission
1131 * @param overwrite if a file with this name already exists, then if true,
1132 * the file will be overwritten, and if false an error will be thrown.
1133 * @param bufferSize the size of the buffer to be used.
1134 * @param replication required block replication for the file.
1135 * @param blockSize
1136 * @param progress
1137 * @throws IOException
1138 * @see #setPermission(Path, FsPermission)
1139 * @deprecated API only for 0.20-append
1140 */
1141 @Deprecated
1142 public FSDataOutputStream createNonRecursive(Path f, FsPermission permission,
1143 boolean overwrite, int bufferSize, short replication, long blockSize,
1144 Progressable progress) throws IOException {
1145 return createNonRecursive(f, permission,
1146 overwrite ? EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE)
1147 : EnumSet.of(CreateFlag.CREATE), bufferSize,
1148 replication, blockSize, progress);
1149 }
1150
1151 /**
1152 * Opens an FSDataOutputStream at the indicated Path with write-progress
1153 * reporting. Same as create(), except fails if parent directory doesn't
1154 * already exist.
1155 * @param f the file name to open
1156 * @param permission
1157 * @param flags {@link CreateFlag}s to use for this stream.
1158 * @param bufferSize the size of the buffer to be used.
1159 * @param replication required block replication for the file.
1160 * @param blockSize
1161 * @param progress
1162 * @throws IOException
1163 * @see #setPermission(Path, FsPermission)
1164 * @deprecated API only for 0.20-append
1165 */
1166 @Deprecated
1167 public FSDataOutputStream createNonRecursive(Path f, FsPermission permission,
1168 EnumSet<CreateFlag> flags, int bufferSize, short replication, long blockSize,
1169 Progressable progress) throws IOException {
1170 throw new IOException("createNonRecursive unsupported for this filesystem "
1171 + this.getClass());
1172 }
1173
1174 /**
1175 * Creates the given Path as a brand-new zero-length file. If
1176 * create fails, or if it already existed, return false.
1177 *
1178 * @param f path to use for create
1179 */
1180 public boolean createNewFile(Path f) throws IOException {
1181 if (exists(f)) {
1182 return false;
1183 } else {
1184 create(f, false, getConf().getInt("io.file.buffer.size", 4096)).close();
1185 return true;
1186 }
1187 }
1188
1189 /**
1190 * Append to an existing file (optional operation).
1191 * Same as append(f, getConf().getInt("io.file.buffer.size", 4096), null)
1192 * @param f the existing file to be appended.
1193 * @throws IOException
1194 */
1195 public FSDataOutputStream append(Path f) throws IOException {
1196 return append(f, getConf().getInt("io.file.buffer.size", 4096), null);
1197 }
1198 /**
1199 * Append to an existing file (optional operation).
1200 * Same as append(f, bufferSize, null).
1201 * @param f the existing file to be appended.
1202 * @param bufferSize the size of the buffer to be used.
1203 * @throws IOException
1204 */
1205 public FSDataOutputStream append(Path f, int bufferSize) throws IOException {
1206 return append(f, bufferSize, null);
1207 }
1208
1209 /**
1210 * Append to an existing file (optional operation).
1211 * @param f the existing file to be appended.
1212 * @param bufferSize the size of the buffer to be used.
1213 * @param progress for reporting progress if it is not null.
1214 * @throws IOException
1215 */
1216 public abstract FSDataOutputStream append(Path f, int bufferSize,
1217 Progressable progress) throws IOException;
1218
1219 /**
1220 * Concat existing files together.
1221 * @param trg the path to the target destination.
1222 * @param psrcs the paths to the sources to use for the concatenation.
1223 * @throws IOException
1224 */
1225 public void concat(final Path trg, final Path [] psrcs) throws IOException {
1226 throw new UnsupportedOperationException("Not implemented by the " +
1227 getClass().getSimpleName() + " FileSystem implementation");
1228 }
1229
1230 /**
1231 * Get replication.
1232 *
1233 * @deprecated Use getFileStatus() instead
1234 * @param src file name
1235 * @return file replication
1236 * @throws IOException
1237 */
1238 @Deprecated
1239 public short getReplication(Path src) throws IOException {
1240 return getFileStatus(src).getReplication();
1241 }
1242
1243 /**
1244 * Set replication for an existing file.
1245 *
1246 * @param src file name
1247 * @param replication new replication
1248 * @throws IOException
1249 * @return true if successful;
1250 * false if file does not exist or is a directory
1251 */
1252 public boolean setReplication(Path src, short replication)
1253 throws IOException {
1254 return true;
1255 }
1256
1257 /**
1258 * Renames Path src to Path dst. Can take place on local fs
1259 * or remote DFS.
1260 * @param src path to be renamed
1261 * @param dst new path after rename
1262 * @throws IOException on failure
1263 * @return true if rename is successful
1264 */
1265 public abstract boolean rename(Path src, Path dst) throws IOException;
1266
1267 /**
1268 * Renames Path src to Path dst
1269 * <ul>
1270 * <li
1271 * <li>Fails if src is a file and dst is a directory.
1272 * <li>Fails if src is a directory and dst is a file.
1273 * <li>Fails if the parent of dst does not exist or is a file.
1274 * </ul>
1275 * <p>
1276 * If OVERWRITE option is not passed as an argument, rename fails
1277 * if the dst already exists.
1278 * <p>
1279 * If OVERWRITE option is passed as an argument, rename overwrites
1280 * the dst if it is a file or an empty directory. Rename fails if dst is
1281 * a non-empty directory.
1282 * <p>
1283 * Note that atomicity of rename is dependent on the file system
1284 * implementation. Please refer to the file system documentation for
1285 * details. This default implementation is non atomic.
1286 * <p>
1287 * This method is deprecated since it is a temporary method added to
1288 * support the transition from FileSystem to FileContext for user
1289 * applications.
1290 *
1291 * @param src path to be renamed
1292 * @param dst new path after rename
1293 * @throws IOException on failure
1294 */
1295 @Deprecated
1296 protected void rename(final Path src, final Path dst,
1297 final Rename... options) throws IOException {
1298 // Default implementation
1299 final FileStatus srcStatus = getFileLinkStatus(src);
1300 if (srcStatus == null) {
1301 throw new FileNotFoundException("rename source " + src + " not found.");
1302 }
1303
1304 boolean overwrite = false;
1305 if (null != options) {
1306 for (Rename option : options) {
1307 if (option == Rename.OVERWRITE) {
1308 overwrite = true;
1309 }
1310 }
1311 }
1312
1313 FileStatus dstStatus;
1314 try {
1315 dstStatus = getFileLinkStatus(dst);
1316 } catch (IOException e) {
1317 dstStatus = null;
1318 }
1319 if (dstStatus != null) {
1320 if (srcStatus.isDirectory() != dstStatus.isDirectory()) {
1321 throw new IOException("Source " + src + " Destination " + dst
1322 + " both should be either file or directory");
1323 }
1324 if (!overwrite) {
1325 throw new FileAlreadyExistsException("rename destination " + dst
1326 + " already exists.");
1327 }
1328 // Delete the destination that is a file or an empty directory
1329 if (dstStatus.isDirectory()) {
1330 FileStatus[] list = listStatus(dst);
1331 if (list != null && list.length != 0) {
1332 throw new IOException(
1333 "rename cannot overwrite non empty destination directory " + dst);
1334 }
1335 }
1336 delete(dst, false);
1337 } else {
1338 final Path parent = dst.getParent();
1339 final FileStatus parentStatus = getFileStatus(parent);
1340 if (parentStatus == null) {
1341 throw new FileNotFoundException("rename destination parent " + parent
1342 + " not found.");
1343 }
1344 if (!parentStatus.isDirectory()) {
1345 throw new ParentNotDirectoryException("rename destination parent " + parent
1346 + " is a file.");
1347 }
1348 }
1349 if (!rename(src, dst)) {
1350 throw new IOException("rename from " + src + " to " + dst + " failed.");
1351 }
1352 }
1353
1354 /**
1355 * Delete a file
1356 * @deprecated Use {@link #delete(Path, boolean)} instead.
1357 */
1358 @Deprecated
1359 public boolean delete(Path f) throws IOException {
1360 return delete(f, true);
1361 }
1362
1363 /** Delete a file.
1364 *
1365 * @param f the path to delete.
1366 * @param recursive if path is a directory and set to
1367 * true, the directory is deleted else throws an exception. In
1368 * case of a file the recursive can be set to either true or false.
1369 * @return true if delete is successful else false.
1370 * @throws IOException
1371 */
1372 public abstract boolean delete(Path f, boolean recursive) throws IOException;
1373
1374 /**
1375 * Mark a path to be deleted when FileSystem is closed.
1376 * When the JVM shuts down,
1377 * all FileSystem objects will be closed automatically.
1378 * Then,
1379 * the marked path will be deleted as a result of closing the FileSystem.
1380 *
1381 * The path has to exist in the file system.
1382 *
1383 * @param f the path to delete.
1384 * @return true if deleteOnExit is successful, otherwise false.
1385 * @throws IOException
1386 */
1387 public boolean deleteOnExit(Path f) throws IOException {
1388 if (!exists(f)) {
1389 return false;
1390 }
1391 synchronized (deleteOnExit) {
1392 deleteOnExit.add(f);
1393 }
1394 return true;
1395 }
1396
1397 /**
1398 * Cancel the deletion of the path when the FileSystem is closed
1399 * @param f the path to cancel deletion
1400 */
1401 public boolean cancelDeleteOnExit(Path f) {
1402 synchronized (deleteOnExit) {
1403 return deleteOnExit.remove(f);
1404 }
1405 }
1406
1407 /**
1408 * Delete all files that were marked as delete-on-exit. This recursively
1409 * deletes all files in the specified paths.
1410 */
1411 protected void processDeleteOnExit() {
1412 synchronized (deleteOnExit) {
1413 for (Iterator<Path> iter = deleteOnExit.iterator(); iter.hasNext();) {
1414 Path path = iter.next();
1415 try {
1416 if (exists(path)) {
1417 delete(path, true);
1418 }
1419 }
1420 catch (IOException e) {
1421 LOG.info("Ignoring failure to deleteOnExit for path " + path);
1422 }
1423 iter.remove();
1424 }
1425 }
1426 }
1427
1428 /** Check if exists.
1429 * @param f source file
1430 */
1431 public boolean exists(Path f) throws IOException {
1432 try {
1433 return getFileStatus(f) != null;
1434 } catch (FileNotFoundException e) {
1435 return false;
1436 }
1437 }
1438
1439 /** True iff the named path is a directory.
1440 * Note: Avoid using this method. Instead reuse the FileStatus
1441 * returned by getFileStatus() or listStatus() methods.
1442 * @param f path to check
1443 */
1444 public boolean isDirectory(Path f) throws IOException {
1445 try {
1446 return getFileStatus(f).isDirectory();
1447 } catch (FileNotFoundException e) {
1448 return false; // f does not exist
1449 }
1450 }
1451
1452 /** True iff the named path is a regular file.
1453 * Note: Avoid using this method. Instead reuse the FileStatus
1454 * returned by getFileStatus() or listStatus() methods.
1455 * @param f path to check
1456 */
1457 public boolean isFile(Path f) throws IOException {
1458 try {
1459 return getFileStatus(f).isFile();
1460 } catch (FileNotFoundException e) {
1461 return false; // f does not exist
1462 }
1463 }
1464
1465 /** The number of bytes in a file. */
1466 /** @deprecated Use getFileStatus() instead */
1467 @Deprecated
1468 public long getLength(Path f) throws IOException {
1469 return getFileStatus(f).getLen();
1470 }
1471
1472 /** Return the {@link ContentSummary} of a given {@link Path}.
1473 * @param f path to use
1474 */
1475 public ContentSummary getContentSummary(Path f) throws IOException {
1476 FileStatus status = getFileStatus(f);
1477 if (status.isFile()) {
1478 // f is a file
1479 return new ContentSummary(status.getLen(), 1, 0);
1480 }
1481 // f is a directory
1482 long[] summary = {0, 0, 1};
1483 for(FileStatus s : listStatus(f)) {
1484 ContentSummary c = s.isDirectory() ? getContentSummary(s.getPath()) :
1485 new ContentSummary(s.getLen(), 1, 0);
1486 summary[0] += c.getLength();
1487 summary[1] += c.getFileCount();
1488 summary[2] += c.getDirectoryCount();
1489 }
1490 return new ContentSummary(summary[0], summary[1], summary[2]);
1491 }
1492
1493 final private static PathFilter DEFAULT_FILTER = new PathFilter() {
1494 @Override
1495 public boolean accept(Path file) {
1496 return true;
1497 }
1498 };
1499
1500 /**
1501 * List the statuses of the files/directories in the given path if the path is
1502 * a directory.
1503 *
1504 * @param f given path
1505 * @return the statuses of the files/directories in the given patch
1506 * @throws FileNotFoundException when the path does not exist;
1507 * IOException see specific implementation
1508 */
1509 public abstract FileStatus[] listStatus(Path f) throws FileNotFoundException,
1510 IOException;
1511
1512 /*
1513 * Filter files/directories in the given path using the user-supplied path
1514 * filter. Results are added to the given array <code>results</code>.
1515 */
1516 private void listStatus(ArrayList<FileStatus> results, Path f,
1517 PathFilter filter) throws FileNotFoundException, IOException {
1518 FileStatus listing[] = listStatus(f);
1519 if (listing == null) {
1520 throw new IOException("Error accessing " + f);
1521 }
1522
1523 for (int i = 0; i < listing.length; i++) {
1524 if (filter.accept(listing[i].getPath())) {
1525 results.add(listing[i]);
1526 }
1527 }
1528 }
1529
1530 /**
1531 * @return an iterator over the corrupt files under the given path
1532 * (may contain duplicates if a file has more than one corrupt block)
1533 * @throws IOException
1534 */
1535 public RemoteIterator<Path> listCorruptFileBlocks(Path path)
1536 throws IOException {
1537 throw new UnsupportedOperationException(getClass().getCanonicalName() +
1538 " does not support" +
1539 " listCorruptFileBlocks");
1540 }
1541
1542 /**
1543 * Filter files/directories in the given path using the user-supplied path
1544 * filter.
1545 *
1546 * @param f
1547 * a path name
1548 * @param filter
1549 * the user-supplied path filter
1550 * @return an array of FileStatus objects for the files under the given path
1551 * after applying the filter
1552 * @throws FileNotFoundException when the path does not exist;
1553 * IOException see specific implementation
1554 */
1555 public FileStatus[] listStatus(Path f, PathFilter filter)
1556 throws FileNotFoundException, IOException {
1557 ArrayList<FileStatus> results = new ArrayList<FileStatus>();
1558 listStatus(results, f, filter);
1559 return results.toArray(new FileStatus[results.size()]);
1560 }
1561
1562 /**
1563 * Filter files/directories in the given list of paths using default
1564 * path filter.
1565 *
1566 * @param files
1567 * a list of paths
1568 * @return a list of statuses for the files under the given paths after
1569 * applying the filter default Path filter
1570 * @throws FileNotFoundException when the path does not exist;
1571 * IOException see specific implementation
1572 */
1573 public FileStatus[] listStatus(Path[] files)
1574 throws FileNotFoundException, IOException {
1575 return listStatus(files, DEFAULT_FILTER);
1576 }
1577
1578 /**
1579 * Filter files/directories in the given list of paths using user-supplied
1580 * path filter.
1581 *
1582 * @param files
1583 * a list of paths
1584 * @param filter
1585 * the user-supplied path filter
1586 * @return a list of statuses for the files under the given paths after
1587 * applying the filter
1588 * @throws FileNotFoundException when the path does not exist;
1589 * IOException see specific implementation
1590 */
1591 public FileStatus[] listStatus(Path[] files, PathFilter filter)
1592 throws FileNotFoundException, IOException {
1593 ArrayList<FileStatus> results = new ArrayList<FileStatus>();
1594 for (int i = 0; i < files.length; i++) {
1595 listStatus(results, files[i], filter);
1596 }
1597 return results.toArray(new FileStatus[results.size()]);
1598 }
1599
1600 /**
1601 * <p>Return all the files that match filePattern and are not checksum
1602 * files. Results are sorted by their names.
1603 *
1604 * <p>
1605 * A filename pattern is composed of <i>regular</i> characters and
1606 * <i>special pattern matching</i> characters, which are:
1607 *
1608 * <dl>
1609 * <dd>
1610 * <dl>
1611 * <p>
1612 * <dt> <tt> ? </tt>
1613 * <dd> Matches any single character.
1614 *
1615 * <p>
1616 * <dt> <tt> * </tt>
1617 * <dd> Matches zero or more characters.
1618 *
1619 * <p>
1620 * <dt> <tt> [<i>abc</i>] </tt>
1621 * <dd> Matches a single character from character set
1622 * <tt>{<i>a,b,c</i>}</tt>.
1623 *
1624 * <p>
1625 * <dt> <tt> [<i>a</i>-<i>b</i>] </tt>
1626 * <dd> Matches a single character from the character range
1627 * <tt>{<i>a...b</i>}</tt>. Note that character <tt><i>a</i></tt> must be
1628 * lexicographically less than or equal to character <tt><i>b</i></tt>.
1629 *
1630 * <p>
1631 * <dt> <tt> [^<i>a</i>] </tt>
1632 * <dd> Matches a single character that is not from character set or range
1633 * <tt>{<i>a</i>}</tt>. Note that the <tt>^</tt> character must occur
1634 * immediately to the right of the opening bracket.
1635 *
1636 * <p>
1637 * <dt> <tt> \<i>c</i> </tt>
1638 * <dd> Removes (escapes) any special meaning of character <i>c</i>.
1639 *
1640 * <p>
1641 * <dt> <tt> {ab,cd} </tt>
1642 * <dd> Matches a string from the string set <tt>{<i>ab, cd</i>} </tt>
1643 *
1644 * <p>
1645 * <dt> <tt> {ab,c{de,fh}} </tt>
1646 * <dd> Matches a string from the string set <tt>{<i>ab, cde, cfh</i>}</tt>
1647 *
1648 * </dl>
1649 * </dd>
1650 * </dl>
1651 *
1652 * @param pathPattern a regular expression specifying a pth pattern
1653
1654 * @return an array of paths that match the path pattern
1655 * @throws IOException
1656 */
1657 public FileStatus[] globStatus(Path pathPattern) throws IOException {
1658 return globStatus(pathPattern, DEFAULT_FILTER);
1659 }
1660
1661 /**
1662 * Return an array of FileStatus objects whose path names match pathPattern
1663 * and is accepted by the user-supplied path filter. Results are sorted by
1664 * their path names.
1665 * Return null if pathPattern has no glob and the path does not exist.
1666 * Return an empty array if pathPattern has a glob and no path matches it.
1667 *
1668 * @param pathPattern
1669 * a regular expression specifying the path pattern
1670 * @param filter
1671 * a user-supplied path filter
1672 * @return an array of FileStatus objects
1673 * @throws IOException if any I/O error occurs when fetching file status
1674 */
1675 public FileStatus[] globStatus(Path pathPattern, PathFilter filter)
1676 throws IOException {
1677 String filename = pathPattern.toUri().getPath();
1678 List<FileStatus> allMatches = null;
1679
1680 List<String> filePatterns = GlobExpander.expand(filename);
1681 for (String filePattern : filePatterns) {
1682 Path path = new Path(filePattern.isEmpty() ? Path.CUR_DIR : filePattern);
1683 List<FileStatus> matches = globStatusInternal(path, filter);
1684 if (matches != null) {
1685 if (allMatches == null) {
1686 allMatches = matches;
1687 } else {
1688 allMatches.addAll(matches);
1689 }
1690 }
1691 }
1692
1693 FileStatus[] results = null;
1694 if (allMatches != null) {
1695 results = allMatches.toArray(new FileStatus[allMatches.size()]);
1696 } else if (filePatterns.size() > 1) {
1697 // no matches with multiple expansions is a non-matching glob
1698 results = new FileStatus[0];
1699 }
1700 return results;
1701 }
1702
1703 // sort gripes because FileStatus Comparable isn't parameterized...
1704 @SuppressWarnings("unchecked")
1705 private List<FileStatus> globStatusInternal(Path pathPattern,
1706 PathFilter filter) throws IOException {
1707 boolean patternHasGlob = false; // pathPattern has any globs
1708 List<FileStatus> matches = new ArrayList<FileStatus>();
1709
1710 // determine starting point
1711 int level = 0;
1712 String baseDir = Path.CUR_DIR;
1713 if (pathPattern.isAbsolute()) {
1714 level = 1; // need to skip empty item at beginning of split list
1715 baseDir = Path.SEPARATOR;
1716 }
1717
1718 // parse components and determine if it's a glob
1719 String[] components = null;
1720 GlobFilter[] filters = null;
1721 String filename = pathPattern.toUri().getPath();
1722 if (!filename.isEmpty() && !Path.SEPARATOR.equals(filename)) {
1723 components = filename.split(Path.SEPARATOR);
1724 filters = new GlobFilter[components.length];
1725 for (int i=level; i < components.length; i++) {
1726 filters[i] = new GlobFilter(components[i]);
1727 patternHasGlob |= filters[i].hasPattern();
1728 }
1729 if (!patternHasGlob) {
1730 baseDir = unquotePathComponent(filename);
1731 components = null; // short through to filter check
1732 }
1733 }
1734
1735 // seed the parent directory path, return if it doesn't exist
1736 try {
1737 matches.add(getFileStatus(new Path(baseDir)));
1738 } catch (FileNotFoundException e) {
1739 return patternHasGlob ? matches : null;
1740 }
1741
1742 // skip if there are no components other than the basedir
1743 if (components != null) {
1744 // iterate through each path component
1745 for (int i=level; (i < components.length) && !matches.isEmpty(); i++) {
1746 List<FileStatus> children = new ArrayList<FileStatus>();
1747 for (FileStatus match : matches) {
1748 // don't look for children in a file matched by a glob
1749 if (!match.isDirectory()) {
1750 continue;
1751 }
1752 try {
1753 if (filters[i].hasPattern()) {
1754 // get all children matching the filter
1755 FileStatus[] statuses = listStatus(match.getPath(), filters[i]);
1756 children.addAll(Arrays.asList(statuses));
1757 } else {
1758 // the component does not have a pattern
1759 String component = unquotePathComponent(components[i]);
1760 Path child = new Path(match.getPath(), component);
1761 children.add(getFileStatus(child));
1762 }
1763 } catch (FileNotFoundException e) {
1764 // don't care
1765 }
1766 }
1767 matches = children;
1768 }
1769 }
1770 // remove anything that didn't match the filter
1771 if (!matches.isEmpty()) {
1772 Iterator<FileStatus> iter = matches.iterator();
1773 while (iter.hasNext()) {
1774 if (!filter.accept(iter.next().getPath())) {
1775 iter.remove();
1776 }
1777 }
1778 }
1779 // no final paths, if there were any globs return empty list
1780 if (matches.isEmpty()) {
1781 return patternHasGlob ? matches : null;
1782 }
1783 Collections.sort(matches);
1784 return matches;
1785 }
1786
1787 /**
1788 * The glob filter builds a regexp per path component. If the component
1789 * does not contain a shell metachar, then it falls back to appending the
1790 * raw string to the list of built up paths. This raw path needs to have
1791 * the quoting removed. Ie. convert all occurances of "\X" to "X"
1792 * @param name of the path component
1793 * @return the unquoted path component
1794 */
1795 private String unquotePathComponent(String name) {
1796 return name.replaceAll("\\\\(.)", "$1");
1797 }
1798
1799 /**
1800 * List the statuses of the files/directories in the given path if the path is
1801 * a directory.
1802 * Return the file's status and block locations If the path is a file.
1803 *
1804 * If a returned status is a file, it contains the file's block locations.
1805 *
1806 * @param f is the path
1807 *
1808 * @return an iterator that traverses statuses of the files/directories
1809 * in the given path
1810 *
1811 * @throws FileNotFoundException If <code>f</code> does not exist
1812 * @throws IOException If an I/O error occurred
1813 */
1814 public RemoteIterator<LocatedFileStatus> listLocatedStatus(final Path f)
1815 throws FileNotFoundException, IOException {
1816 return listLocatedStatus(f, DEFAULT_FILTER);
1817 }
1818
1819 /**
1820 * Listing a directory
1821 * The returned results include its block location if it is a file
1822 * The results are filtered by the given path filter
1823 * @param f a path
1824 * @param filter a path filter
1825 * @return an iterator that traverses statuses of the files/directories
1826 * in the given path
1827 * @throws FileNotFoundException if <code>f</code> does not exist
1828 * @throws IOException if any I/O error occurred
1829 */
1830 protected RemoteIterator<LocatedFileStatus> listLocatedStatus(final Path f,
1831 final PathFilter filter)
1832 throws FileNotFoundException, IOException {
1833 return new RemoteIterator<LocatedFileStatus>() {
1834 private final FileStatus[] stats = listStatus(f, filter);
1835 private int i = 0;
1836
1837 @Override
1838 public boolean hasNext() {
1839 return i<stats.length;
1840 }
1841
1842 @Override
1843 public LocatedFileStatus next() throws IOException {
1844 if (!hasNext()) {
1845 throw new NoSuchElementException("No more entry in " + f);
1846 }
1847 FileStatus result = stats[i++];
1848 BlockLocation[] locs = result.isFile() ?
1849 getFileBlockLocations(result.getPath(), 0, result.getLen()) :
1850 null;
1851 return new LocatedFileStatus(result, locs);
1852 }
1853 };
1854 }
1855
1856 /**
1857 * List the statuses and block locations of the files in the given path.
1858 *
1859 * If the path is a directory,
1860 * if recursive is false, returns files in the directory;
1861 * if recursive is true, return files in the subtree rooted at the path.
1862 * If the path is a file, return the file's status and block locations.
1863 *
1864 * @param f is the path
1865 * @param recursive if the subdirectories need to be traversed recursively
1866 *
1867 * @return an iterator that traverses statuses of the files
1868 *
1869 * @throws FileNotFoundException when the path does not exist;
1870 * IOException see specific implementation
1871 */
1872 public RemoteIterator<LocatedFileStatus> listFiles(
1873 final Path f, final boolean recursive)
1874 throws FileNotFoundException, IOException {
1875 return new RemoteIterator<LocatedFileStatus>() {
1876 private Stack<RemoteIterator<LocatedFileStatus>> itors =
1877 new Stack<RemoteIterator<LocatedFileStatus>>();
1878 private RemoteIterator<LocatedFileStatus> curItor =
1879 listLocatedStatus(f);
1880 private LocatedFileStatus curFile;
1881
1882 @Override
1883 public boolean hasNext() throws IOException {
1884 while (curFile == null) {
1885 if (curItor.hasNext()) {
1886 handleFileStat(curItor.next());
1887 } else if (!itors.empty()) {
1888 curItor = itors.pop();
1889 } else {
1890 return false;
1891 }
1892 }
1893 return true;
1894 }
1895
1896 /**
1897 * Process the input stat.
1898 * If it is a file, return the file stat.
1899 * If it is a directory, traverse the directory if recursive is true;
1900 * ignore it if recursive is false.
1901 * @param stat input status
1902 * @throws IOException if any IO error occurs
1903 */
1904 private void handleFileStat(LocatedFileStatus stat) throws IOException {
1905 if (stat.isFile()) { // file
1906 curFile = stat;
1907 } else if (recursive) { // directory
1908 itors.push(curItor);
1909 curItor = listLocatedStatus(stat.getPath());
1910 }
1911 }
1912
1913 @Override
1914 public LocatedFileStatus next() throws IOException {
1915 if (hasNext()) {
1916 LocatedFileStatus result = curFile;
1917 curFile = null;
1918 return result;
1919 }
1920 throw new java.util.NoSuchElementException("No more entry in " + f);
1921 }
1922 };
1923 }
1924
1925 /** Return the current user's home directory in this filesystem.
1926 * The default implementation returns "/user/$USER/".
1927 */
1928 public Path getHomeDirectory() {
1929 return this.makeQualified(
1930 new Path("/user/"+System.getProperty("user.name")));
1931 }
1932
1933
1934 /**
1935 * Set the current working directory for the given file system. All relative
1936 * paths will be resolved relative to it.
1937 *
1938 * @param new_dir
1939 */
1940 public abstract void setWorkingDirectory(Path new_dir);
1941
1942 /**
1943 * Get the current working directory for the given file system
1944 * @return the directory pathname
1945 */
1946 public abstract Path getWorkingDirectory();
1947
1948
1949 /**
1950 * Note: with the new FilesContext class, getWorkingDirectory()
1951 * will be removed.
1952 * The working directory is implemented in FilesContext.
1953 *
1954 * Some file systems like LocalFileSystem have an initial workingDir
1955 * that we use as the starting workingDir. For other file systems
1956 * like HDFS there is no built in notion of an initial workingDir.
1957 *
1958 * @return if there is built in notion of workingDir then it
1959 * is returned; else a null is returned.
1960 */
1961 protected Path getInitialWorkingDirectory() {
1962 return null;
1963 }
1964
1965 /**
1966 * Call {@link #mkdirs(Path, FsPermission)} with default permission.
1967 */
1968 public boolean mkdirs(Path f) throws IOException {
1969 return mkdirs(f, FsPermission.getDirDefault());
1970 }
1971
1972 /**
1973 * Make the given file and all non-existent parents into
1974 * directories. Has the semantics of Unix 'mkdir -p'.
1975 * Existence of the directory hierarchy is not an error.
1976 * @param f path to create
1977 * @param permission to apply to f
1978 */
1979 public abstract boolean mkdirs(Path f, FsPermission permission
1980 ) throws IOException;
1981
1982 /**
1983 * The src file is on the local disk. Add it to FS at
1984 * the given dst name and the source is kept intact afterwards
1985 * @param src path
1986 * @param dst path
1987 */
1988 public void copyFromLocalFile(Path src, Path dst)
1989 throws IOException {
1990 copyFromLocalFile(false, src, dst);
1991 }
1992
1993 /**
1994 * The src files is on the local disk. Add it to FS at
1995 * the given dst name, removing the source afterwards.
1996 * @param srcs path
1997 * @param dst path
1998 */
1999 public void moveFromLocalFile(Path[] srcs, Path dst)
2000 throws IOException {
2001 copyFromLocalFile(true, true, srcs, dst);
2002 }
2003
2004 /**
2005 * The src file is on the local disk. Add it to FS at
2006 * the given dst name, removing the source afterwards.
2007 * @param src path
2008 * @param dst path
2009 */
2010 public void moveFromLocalFile(Path src, Path dst)
2011 throws IOException {
2012 copyFromLocalFile(true, src, dst);
2013 }
2014
2015 /**
2016 * The src file is on the local disk. Add it to FS at
2017 * the given dst name.
2018 * delSrc indicates if the source should be removed
2019 * @param delSrc whether to delete the src
2020 * @param src path
2021 * @param dst path
2022 */
2023 public void copyFromLocalFile(boolean delSrc, Path src, Path dst)
2024 throws IOException {
2025 copyFromLocalFile(delSrc, true, src, dst);
2026 }
2027
2028 /**
2029 * The src files are on the local disk. Add it to FS at
2030 * the given dst name.
2031 * delSrc indicates if the source should be removed
2032 * @param delSrc whether to delete the src
2033 * @param overwrite whether to overwrite an existing file
2034 * @param srcs array of paths which are source
2035 * @param dst path
2036 */
2037 public void copyFromLocalFile(boolean delSrc, boolean overwrite,
2038 Path[] srcs, Path dst)
2039 throws IOException {
2040 Configuration conf = getConf();
2041 FileUtil.copy(getLocal(conf), srcs, this, dst, delSrc, overwrite, conf);
2042 }
2043
2044 /**
2045 * The src file is on the local disk. Add it to FS at
2046 * the given dst name.
2047 * delSrc indicates if the source should be removed
2048 * @param delSrc whether to delete the src
2049 * @param overwrite whether to overwrite an existing file
2050 * @param src path
2051 * @param dst path
2052 */
2053 public void copyFromLocalFile(boolean delSrc, boolean overwrite,
2054 Path src, Path dst)
2055 throws IOException {
2056 Configuration conf = getConf();
2057 FileUtil.copy(getLocal(conf), src, this, dst, delSrc, overwrite, conf);
2058 }
2059
2060 /**
2061 * The src file is under FS, and the dst is on the local disk.
2062 * Copy it from FS control to the local dst name.
2063 * @param src path
2064 * @param dst path
2065 */
2066 public void copyToLocalFile(Path src, Path dst) throws IOException {
2067 copyToLocalFile(false, src, dst);
2068 }
2069
2070 /**
2071 * The src file is under FS, and the dst is on the local disk.
2072 * Copy it from FS control to the local dst name.
2073 * Remove the source afterwards
2074 * @param src path
2075 * @param dst path
2076 */
2077 public void moveToLocalFile(Path src, Path dst) throws IOException {
2078 copyToLocalFile(true, src, dst);
2079 }
2080
2081 /**
2082 * The src file is under FS, and the dst is on the local disk.
2083 * Copy it from FS control to the local dst name.
2084 * delSrc indicates if the src will be removed or not.
2085 * @param delSrc whether to delete the src
2086 * @param src path
2087 * @param dst path
2088 */
2089 public void copyToLocalFile(boolean delSrc, Path src, Path dst)
2090 throws IOException {
2091 copyToLocalFile(delSrc, src, dst, false);
2092 }
2093
2094 /**
2095 * The src file is under FS, and the dst is on the local disk. Copy it from FS
2096 * control to the local dst name. delSrc indicates if the src will be removed
2097 * or not. useRawLocalFileSystem indicates whether to use RawLocalFileSystem
2098 * as local file system or not. RawLocalFileSystem is non crc file system.So,
2099 * It will not create any crc files at local.
2100 *
2101 * @param delSrc
2102 * whether to delete the src
2103 * @param src
2104 * path
2105 * @param dst
2106 * path
2107 * @param useRawLocalFileSystem
2108 * whether to use RawLocalFileSystem as local file system or not.
2109 *
2110 * @throws IOException
2111 * - if any IO error
2112 */
2113 public void copyToLocalFile(boolean delSrc, Path src, Path dst,
2114 boolean useRawLocalFileSystem) throws IOException {
2115 Configuration conf = getConf();
2116 FileSystem local = null;
2117 if (useRawLocalFileSystem) {
2118 local = getLocal(conf).getRawFileSystem();
2119 } else {
2120 local = getLocal(conf);
2121 }
2122 FileUtil.copy(this, src, local, dst, delSrc, conf);
2123 }
2124
2125 /**
2126 * Returns a local File that the user can write output to. The caller
2127 * provides both the eventual FS target name and the local working
2128 * file. If the FS is local, we write directly into the target. If
2129 * the FS is remote, we write into the tmp local area.
2130 * @param fsOutputFile path of output file
2131 * @param tmpLocalFile path of local tmp file
2132 */
2133 public Path startLocalOutput(Path fsOutputFile, Path tmpLocalFile)
2134 throws IOException {
2135 return tmpLocalFile;
2136 }
2137
2138 /**
2139 * Called when we're all done writing to the target. A local FS will
2140 * do nothing, because we've written to exactly the right place. A remote
2141 * FS will copy the contents of tmpLocalFile to the correct target at
2142 * fsOutputFile.
2143 * @param fsOutputFile path of output file
2144 * @param tmpLocalFile path to local tmp file
2145 */
2146 public void completeLocalOutput(Path fsOutputFile, Path tmpLocalFile)
2147 throws IOException {
2148 moveFromLocalFile(tmpLocalFile, fsOutputFile);
2149 }
2150
2151 /**
2152 * No more filesystem operations are needed. Will
2153 * release any held locks.
2154 */
2155 @Override
2156 public void close() throws IOException {
2157 // delete all files that were marked as delete-on-exit.
2158 processDeleteOnExit();
2159 CACHE.remove(this.key, this);
2160 }
2161
2162 /** Return the total size of all files in the filesystem.*/
2163 public long getUsed() throws IOException{
2164 long used = 0;
2165 FileStatus[] files = listStatus(new Path("/"));
2166 for(FileStatus file:files){
2167 used += file.getLen();
2168 }
2169 return used;
2170 }
2171
2172 /**
2173 * Get the block size for a particular file.
2174 * @param f the filename
2175 * @return the number of bytes in a block
2176 */
2177 /** @deprecated Use getFileStatus() instead */
2178 @Deprecated
2179 public long getBlockSize(Path f) throws IOException {
2180 return getFileStatus(f).getBlockSize();
2181 }
2182
2183 /**
2184 * Return the number of bytes that large input files should be optimally
2185 * be split into to minimize i/o time.
2186 * @deprecated use {@link #getDefaultBlockSize(Path)} instead
2187 */
2188 @Deprecated
2189 public long getDefaultBlockSize() {
2190 // default to 32MB: large enough to minimize the impact of seeks
2191 return getConf().getLong("fs.local.block.size", 32 * 1024 * 1024);
2192 }
2193
2194 /** Return the number of bytes that large input files should be optimally
2195 * be split into to minimize i/o time. The given path will be used to
2196 * locate the actual filesystem. The full path does not have to exist.
2197 * @param f path of file
2198 * @return the default block size for the path's filesystem
2199 */
2200 public long getDefaultBlockSize(Path f) {
2201 return getDefaultBlockSize();
2202 }
2203
2204 /**
2205 * Get the default replication.
2206 * @deprecated use {@link #getDefaultReplication(Path)} instead
2207 */
2208 @Deprecated
2209 public short getDefaultReplication() { return 1; }
2210
2211 /**
2212 * Get the default replication for a path. The given path will be used to
2213 * locate the actual filesystem. The full path does not have to exist.
2214 * @param path of the file
2215 * @return default replication for the path's filesystem
2216 */
2217 public short getDefaultReplication(Path path) {
2218 return getDefaultReplication();
2219 }
2220
2221 /**
2222 * Return a file status object that represents the path.
2223 * @param f The path we want information from
2224 * @return a FileStatus object
2225 * @throws FileNotFoundException when the path does not exist;
2226 * IOException see specific implementation
2227 */
2228 public abstract FileStatus getFileStatus(Path f) throws IOException;
2229
2230 /**
2231 * See {@link FileContext#fixRelativePart}
2232 */
2233 protected Path fixRelativePart(Path p) {
2234 if (p.isUriPathAbsolute()) {
2235 return p;
2236 } else {
2237 return new Path(getWorkingDirectory(), p);
2238 }
2239 }
2240
2241 /**
2242 * See {@link FileContext#createSymlink(Path, Path, boolean)}
2243 */
2244 public void createSymlink(final Path target, final Path link,
2245 final boolean createParent) throws AccessControlException,
2246 FileAlreadyExistsException, FileNotFoundException,
2247 ParentNotDirectoryException, UnsupportedFileSystemException,
2248 IOException {
2249 // Supporting filesystems should override this method
2250 throw new UnsupportedOperationException(
2251 "Filesystem does not support symlinks!");
2252 }
2253
2254 /**
2255 * See {@link FileContext#getFileLinkStatus(Path)}
2256 */
2257 public FileStatus getFileLinkStatus(final Path f)
2258 throws AccessControlException, FileNotFoundException,
2259 UnsupportedFileSystemException, IOException {
2260 // Supporting filesystems should override this method
2261 return getFileStatus(f);
2262 }
2263
2264 /**
2265 * See {@link AbstractFileSystem#supportsSymlinks()}
2266 */
2267 public boolean supportsSymlinks() {
2268 return false;
2269 }
2270
2271 /**
2272 * See {@link FileContext#getLinkTarget(Path)}
2273 */
2274 public Path getLinkTarget(Path f) throws IOException {
2275 // Supporting filesystems should override this method
2276 throw new UnsupportedOperationException(
2277 "Filesystem does not support symlinks!");
2278 }
2279
2280 /**
2281 * See {@link AbstractFileSystem#getLinkTarget(Path)}
2282 */
2283 protected Path resolveLink(Path f) throws IOException {
2284 // Supporting filesystems should override this method
2285 throw new UnsupportedOperationException(
2286 "Filesystem does not support symlinks!");
2287 }
2288
2289 /**
2290 * Get the checksum of a file.
2291 *
2292 * @param f The file path
2293 * @return The file checksum. The default return value is null,
2294 * which indicates that no checksum algorithm is implemented
2295 * in the corresponding FileSystem.
2296 */
2297 public FileChecksum getFileChecksum(Path f) throws IOException {
2298 return null;
2299 }
2300
2301 /**
2302 * Set the verify checksum flag. This is only applicable if the
2303 * corresponding FileSystem supports checksum. By default doesn't do anything.
2304 * @param verifyChecksum
2305 */
2306 public void setVerifyChecksum(boolean verifyChecksum) {
2307 //doesn't do anything
2308 }
2309
2310 /**
2311 * Set the write checksum flag. This is only applicable if the
2312 * corresponding FileSystem supports checksum. By default doesn't do anything.
2313 * @param writeChecksum
2314 */
2315 public void setWriteChecksum(boolean writeChecksum) {
2316 //doesn't do anything
2317 }
2318
2319 /**
2320 * Returns a status object describing the use and capacity of the
2321 * file system. If the file system has multiple partitions, the
2322 * use and capacity of the root partition is reflected.
2323 *
2324 * @return a FsStatus object
2325 * @throws IOException
2326 * see specific implementation
2327 */
2328 public FsStatus getStatus() throws IOException {
2329 return getStatus(null);
2330 }
2331
2332 /**
2333 * Returns a status object describing the use and capacity of the
2334 * file system. If the file system has multiple partitions, the
2335 * use and capacity of the partition pointed to by the specified
2336 * path is reflected.
2337 * @param p Path for which status should be obtained. null means
2338 * the default partition.
2339 * @return a FsStatus object
2340 * @throws IOException
2341 * see specific implementation
2342 */
2343 public FsStatus getStatus(Path p) throws IOException {
2344 return new FsStatus(Long.MAX_VALUE, 0, Long.MAX_VALUE);
2345 }
2346
2347 /**
2348 * Set permission of a path.
2349 * @param p
2350 * @param permission
2351 */
2352 public void setPermission(Path p, FsPermission permission
2353 ) throws IOException {
2354 }
2355
2356 /**
2357 * Set owner of a path (i.e. a file or a directory).
2358 * The parameters username and groupname cannot both be null.
2359 * @param p The path
2360 * @param username If it is null, the original username remains unchanged.
2361 * @param groupname If it is null, the original groupname remains unchanged.
2362 */
2363 public void setOwner(Path p, String username, String groupname
2364 ) throws IOException {
2365 }
2366
2367 /**
2368 * Set access time of a file
2369 * @param p The path
2370 * @param mtime Set the modification time of this file.
2371 * The number of milliseconds since Jan 1, 1970.
2372 * A value of -1 means that this call should not set modification time.
2373 * @param atime Set the access time of this file.
2374 * The number of milliseconds since Jan 1, 1970.
2375 * A value of -1 means that this call should not set access time.
2376 */
2377 public void setTimes(Path p, long mtime, long atime
2378 ) throws IOException {
2379 }
2380
2381 /**
2382 * Create a snapshot with a default name.
2383 * @param path The directory where snapshots will be taken.
2384 * @return the snapshot path.
2385 */
2386 public final Path createSnapshot(Path path) throws IOException {
2387 return createSnapshot(path, null);
2388 }
2389
2390 /**
2391 * Create a snapshot
2392 * @param path The directory where snapshots will be taken.
2393 * @param snapshotName The name of the snapshot
2394 * @return the snapshot path.
2395 */
2396 public Path createSnapshot(Path path, String snapshotName)
2397 throws IOException {
2398 throw new UnsupportedOperationException(getClass().getSimpleName()
2399 + " doesn't support createSnapshot");
2400 }
2401
2402 /**
2403 * Rename a snapshot
2404 * @param path The directory path where the snapshot was taken
2405 * @param snapshotOldName Old name of the snapshot
2406 * @param snapshotNewName New name of the snapshot
2407 * @throws IOException
2408 */
2409 public void renameSnapshot(Path path, String snapshotOldName,
2410 String snapshotNewName) throws IOException {
2411 throw new UnsupportedOperationException(getClass().getSimpleName()
2412 + " doesn't support renameSnapshot");
2413 }
2414
2415 /**
2416 * Delete a snapshot of a directory
2417 * @param path The directory that the to-be-deleted snapshot belongs to
2418 * @param snapshotName The name of the snapshot
2419 */
2420 public void deleteSnapshot(Path path, String snapshotName)
2421 throws IOException {
2422 throw new UnsupportedOperationException(getClass().getSimpleName()
2423 + " doesn't support deleteSnapshot");
2424 }
2425
2426 // making it volatile to be able to do a double checked locking
2427 private volatile static boolean FILE_SYSTEMS_LOADED = false;
2428
2429 private static final Map<String, Class<? extends FileSystem>>
2430 SERVICE_FILE_SYSTEMS = new HashMap<String, Class<? extends FileSystem>>();
2431
2432 private static void loadFileSystems() {
2433 synchronized (FileSystem.class) {
2434 if (!FILE_SYSTEMS_LOADED) {
2435 ServiceLoader<FileSystem> serviceLoader = ServiceLoader.load(FileSystem.class);
2436 for (FileSystem fs : serviceLoader) {
2437 SERVICE_FILE_SYSTEMS.put(fs.getScheme(), fs.getClass());
2438 }
2439 FILE_SYSTEMS_LOADED = true;
2440 }
2441 }
2442 }
2443
2444 public static Class<? extends FileSystem> getFileSystemClass(String scheme,
2445 Configuration conf) throws IOException {
2446 if (!FILE_SYSTEMS_LOADED) {
2447 loadFileSystems();
2448 }
2449 Class<? extends FileSystem> clazz = null;
2450 if (conf != null) {
2451 clazz = (Class<? extends FileSystem>) conf.getClass("fs." + scheme + ".impl", null);
2452 }
2453 if (clazz == null) {
2454 clazz = SERVICE_FILE_SYSTEMS.get(scheme);
2455 }
2456 if (clazz == null) {
2457 throw new IOException("No FileSystem for scheme: " + scheme);
2458 }
2459 return clazz;
2460 }
2461
2462 private static FileSystem createFileSystem(URI uri, Configuration conf
2463 ) throws IOException {
2464 Class<?> clazz = getFileSystemClass(uri.getScheme(), conf);
2465 if (clazz == null) {
2466 throw new IOException("No FileSystem for scheme: " + uri.getScheme());
2467 }
2468 FileSystem fs = (FileSystem)ReflectionUtils.newInstance(clazz, conf);
2469 fs.initialize(uri, conf);
2470 return fs;
2471 }
2472
2473 /** Caching FileSystem objects */
2474 static class Cache {
2475 private final ClientFinalizer clientFinalizer = new ClientFinalizer();
2476
2477 private final Map<Key, FileSystem> map = new HashMap<Key, FileSystem>();
2478 private final Set<Key> toAutoClose = new HashSet<Key>();
2479
2480 /** A variable that makes all objects in the cache unique */
2481 private static AtomicLong unique = new AtomicLong(1);
2482
2483 FileSystem get(URI uri, Configuration conf) throws IOException{
2484 Key key = new Key(uri, conf);
2485 return getInternal(uri, conf, key);
2486 }
2487
2488 /** The objects inserted into the cache using this method are all unique */
2489 FileSystem getUnique(URI uri, Configuration conf) throws IOException{
2490 Key key = new Key(uri, conf, unique.getAndIncrement());
2491 return getInternal(uri, conf, key);
2492 }
2493
2494 private FileSystem getInternal(URI uri, Configuration conf, Key key) throws IOException{
2495 FileSystem fs;
2496 synchronized (this) {
2497 fs = map.get(key);
2498 }
2499 if (fs != null) {
2500 return fs;
2501 }
2502
2503 fs = createFileSystem(uri, conf);
2504 synchronized (this) { // refetch the lock again
2505 FileSystem oldfs = map.get(key);
2506 if (oldfs != null) { // a file system is created while lock is releasing
2507 fs.close(); // close the new file system
2508 return oldfs; // return the old file system
2509 }
2510
2511 // now insert the new file system into the map
2512 if (map.isEmpty()
2513 && !ShutdownHookManager.get().isShutdownInProgress()) {
2514 ShutdownHookManager.get().addShutdownHook(clientFinalizer, SHUTDOWN_HOOK_PRIORITY);
2515 }
2516 fs.key = key;
2517 map.put(key, fs);
2518 if (conf.getBoolean("fs.automatic.close", true)) {
2519 toAutoClose.add(key);
2520 }
2521 return fs;
2522 }
2523 }
2524
2525 synchronized void remove(Key key, FileSystem fs) {
2526 if (map.containsKey(key) && fs == map.get(key)) {
2527 map.remove(key);
2528 toAutoClose.remove(key);
2529 }
2530 }
2531
2532 synchronized void closeAll() throws IOException {
2533 closeAll(false);
2534 }
2535
2536 /**
2537 * Close all FileSystem instances in the Cache.
2538 * @param onlyAutomatic only close those that are marked for automatic closing
2539 */
2540 synchronized void closeAll(boolean onlyAutomatic) throws IOException {
2541 List<IOException> exceptions = new ArrayList<IOException>();
2542
2543 // Make a copy of the keys in the map since we'll be modifying
2544 // the map while iterating over it, which isn't safe.
2545 List<Key> keys = new ArrayList<Key>();
2546 keys.addAll(map.keySet());
2547
2548 for (Key key : keys) {
2549 final FileSystem fs = map.get(key);
2550
2551 if (onlyAutomatic && !toAutoClose.contains(key)) {
2552 continue;
2553 }
2554
2555 //remove from cache
2556 remove(key, fs);
2557
2558 if (fs != null) {
2559 try {
2560 fs.close();
2561 }
2562 catch(IOException ioe) {
2563 exceptions.add(ioe);
2564 }
2565 }
2566 }
2567
2568 if (!exceptions.isEmpty()) {
2569 throw MultipleIOException.createIOException(exceptions);
2570 }
2571 }
2572
2573 private class ClientFinalizer implements Runnable {
2574 @Override
2575 public synchronized void run() {
2576 try {
2577 closeAll(true);
2578 } catch (IOException e) {
2579 LOG.info("FileSystem.Cache.closeAll() threw an exception:\n" + e);
2580 }
2581 }
2582 }
2583
2584 synchronized void closeAll(UserGroupInformation ugi) throws IOException {
2585 List<FileSystem> targetFSList = new ArrayList<FileSystem>();
2586 //Make a pass over the list and collect the filesystems to close
2587 //we cannot close inline since close() removes the entry from the Map
2588 for (Map.Entry<Key, FileSystem> entry : map.entrySet()) {
2589 final Key key = entry.getKey();
2590 final FileSystem fs = entry.getValue();
2591 if (ugi.equals(key.ugi) && fs != null) {
2592 targetFSList.add(fs);
2593 }
2594 }
2595 List<IOException> exceptions = new ArrayList<IOException>();
2596 //now make a pass over the target list and close each
2597 for (FileSystem fs : targetFSList) {
2598 try {
2599 fs.close();
2600 }
2601 catch(IOException ioe) {
2602 exceptions.add(ioe);
2603 }
2604 }
2605 if (!exceptions.isEmpty()) {
2606 throw MultipleIOException.createIOException(exceptions);
2607 }
2608 }
2609
2610 /** FileSystem.Cache.Key */
2611 static class Key {
2612 final String scheme;
2613 final String authority;
2614 final UserGroupInformation ugi;
2615 final long unique; // an artificial way to make a key unique
2616
2617 Key(URI uri, Configuration conf) throws IOException {
2618 this(uri, conf, 0);
2619 }
2620
2621 Key(URI uri, Configuration conf, long unique) throws IOException {
2622 scheme = uri.getScheme()==null?"":uri.getScheme().toLowerCase();
2623 authority = uri.getAuthority()==null?"":uri.getAuthority().toLowerCase();
2624 this.unique = unique;
2625
2626 this.ugi = UserGroupInformation.getCurrentUser();
2627 }
2628
2629 @Override
2630 public int hashCode() {
2631 return (scheme + authority).hashCode() + ugi.hashCode() + (int)unique;
2632 }
2633
2634 static boolean isEqual(Object a, Object b) {
2635 return a == b || (a != null && a.equals(b));
2636 }
2637
2638 @Override
2639 public boolean equals(Object obj) {
2640 if (obj == this) {
2641 return true;
2642 }
2643 if (obj != null && obj instanceof Key) {
2644 Key that = (Key)obj;
2645 return isEqual(this.scheme, that.scheme)
2646 && isEqual(this.authority, that.authority)
2647 && isEqual(this.ugi, that.ugi)
2648 && (this.unique == that.unique);
2649 }
2650 return false;
2651 }
2652
2653 @Override
2654 public String toString() {
2655 return "("+ugi.toString() + ")@" + scheme + "://" + authority;
2656 }
2657 }
2658 }
2659
2660 public static final class Statistics {
2661 private final String scheme;
2662 private AtomicLong bytesRead = new AtomicLong();
2663 private AtomicLong bytesWritten = new AtomicLong();
2664 private AtomicInteger readOps = new AtomicInteger();
2665 private AtomicInteger largeReadOps = new AtomicInteger();
2666 private AtomicInteger writeOps = new AtomicInteger();
2667
2668 public Statistics(String scheme) {
2669 this.scheme = scheme;
2670 }
2671
2672 /**
2673 * Copy constructor.
2674 *
2675 * @param st
2676 * The input Statistics object which is cloned.
2677 */
2678 public Statistics(Statistics st) {
2679 this.scheme = st.scheme;
2680 this.bytesRead = new AtomicLong(st.bytesRead.longValue());
2681 this.bytesWritten = new AtomicLong(st.bytesWritten.longValue());
2682 }
2683
2684 /**
2685 * Increment the bytes read in the statistics
2686 * @param newBytes the additional bytes read
2687 */
2688 public void incrementBytesRead(long newBytes) {
2689 bytesRead.getAndAdd(newBytes);
2690 }
2691
2692 /**
2693 * Increment the bytes written in the statistics
2694 * @param newBytes the additional bytes written
2695 */
2696 public void incrementBytesWritten(long newBytes) {
2697 bytesWritten.getAndAdd(newBytes);
2698 }
2699
2700 /**
2701 * Increment the number of read operations
2702 * @param count number of read operations
2703 */
2704 public void incrementReadOps(int count) {
2705 readOps.getAndAdd(count);
2706 }
2707
2708 /**
2709 * Increment the number of large read operations
2710 * @param count number of large read operations
2711 */
2712 public void incrementLargeReadOps(int count) {
2713 largeReadOps.getAndAdd(count);
2714 }
2715
2716 /**
2717 * Increment the number of write operations
2718 * @param count number of write operations
2719 */
2720 public void incrementWriteOps(int count) {
2721 writeOps.getAndAdd(count);
2722 }
2723
2724 /**
2725 * Get the total number of bytes read
2726 * @return the number of bytes
2727 */
2728 public long getBytesRead() {
2729 return bytesRead.get();
2730 }
2731
2732 /**
2733 * Get the total number of bytes written
2734 * @return the number of bytes
2735 */
2736 public long getBytesWritten() {
2737 return bytesWritten.get();
2738 }
2739
2740 /**
2741 * Get the number of file system read operations such as list files
2742 * @return number of read operations
2743 */
2744 public int getReadOps() {
2745 return readOps.get() + largeReadOps.get();
2746 }
2747
2748 /**
2749 * Get the number of large file system read operations such as list files
2750 * under a large directory
2751 * @return number of large read operations
2752 */
2753 public int getLargeReadOps() {
2754 return largeReadOps.get();
2755 }
2756
2757 /**
2758 * Get the number of file system write operations such as create, append
2759 * rename etc.
2760 * @return number of write operations
2761 */
2762 public int getWriteOps() {
2763 return writeOps.get();
2764 }
2765
2766 @Override
2767 public String toString() {
2768 return bytesRead + " bytes read, " + bytesWritten + " bytes written, "
2769 + readOps + " read ops, " + largeReadOps + " large read ops, "
2770 + writeOps + " write ops";
2771 }
2772
2773 /**
2774 * Reset the counts to 0.
2775 */
2776 public void reset() {
2777 bytesWritten.set(0);
2778 bytesRead.set(0);
2779 readOps.set(0);
2780 largeReadOps.set(0);
2781 writeOps.set(0);
2782 }
2783
2784 /**
2785 * Get the uri scheme associated with this statistics object.
2786 * @return the schema associated with this set of statistics
2787 */
2788 public String getScheme() {
2789 return scheme;
2790 }
2791 }
2792
2793 /**
2794 * Get the Map of Statistics object indexed by URI Scheme.
2795 * @return a Map having a key as URI scheme and value as Statistics object
2796 * @deprecated use {@link #getAllStatistics} instead
2797 */
2798 @Deprecated
2799 public static synchronized Map<String, Statistics> getStatistics() {
2800 Map<String, Statistics> result = new HashMap<String, Statistics>();
2801 for(Statistics stat: statisticsTable.values()) {
2802 result.put(stat.getScheme(), stat);
2803 }
2804 return result;
2805 }
2806
2807 /**
2808 * Return the FileSystem classes that have Statistics
2809 */
2810 public static synchronized List<Statistics> getAllStatistics() {
2811 return new ArrayList<Statistics>(statisticsTable.values());
2812 }
2813
2814 /**
2815 * Get the statistics for a particular file system
2816 * @param cls the class to lookup
2817 * @return a statistics object
2818 */
2819 public static synchronized
2820 Statistics getStatistics(String scheme, Class<? extends FileSystem> cls) {
2821 Statistics result = statisticsTable.get(cls);
2822 if (result == null) {
2823 result = new Statistics(scheme);
2824 statisticsTable.put(cls, result);
2825 }
2826 return result;
2827 }
2828
2829 /**
2830 * Reset all statistics for all file systems
2831 */
2832 public static synchronized void clearStatistics() {
2833 for(Statistics stat: statisticsTable.values()) {
2834 stat.reset();
2835 }
2836 }
2837
2838 /**
2839 * Print all statistics for all file systems
2840 */
2841 public static synchronized
2842 void printStatistics() throws IOException {
2843 for (Map.Entry<Class<? extends FileSystem>, Statistics> pair:
2844 statisticsTable.entrySet()) {
2845 System.out.println(" FileSystem " + pair.getKey().getName() +
2846 ": " + pair.getValue());
2847 }
2848 }
2849
2850 // Symlinks are temporarily disabled - see Hadoop-10020 and HADOOP-10052
2851 private static boolean symlinkEnabled = false;
2852
2853 @Deprecated
2854 @VisibleForTesting
2855 public static boolean isSymlinksEnabled() {
2856 return symlinkEnabled;
2857 }
2858
2859 @Deprecated
2860 @VisibleForTesting
2861 public static void enableSymlinks() {
2862 symlinkEnabled = true;
2863 }
2864
2865 /**
2866 * MapR addition:
2867 * Opens an FSDataInputStream at the indicated fid.
2868 * @param fid the fid to open
2869 * @param ips the list of ip/ports which this fid belongs to
2870 * @param chunkSize the chunkSize of the file corresponding to the fid
2871 * @param fileSize the size of the file corresponding to the fid
2872 */
2873 public FSDataInputStream openFid(String fid, long[] ips,
2874 long chunkSize, long fileSize) throws IOException {
2875 throw new UnsupportedOperationException("See concrete FS for implementation");
2876 }
2877
2878 /**
2879 * MapR addition:
2880 * Opens an FSDataInputStream at the indicated fid.
2881 * @param pfid the parent-fid of the file to open
2882 * @param file the file to be opened
2883 */
2884 public FSDataInputStream openFid(String pfid, String file, long [] ips)
2885 throws IOException {
2886 throw new UnsupportedOperationException("See concrete FS for implementation");
2887 }
2888
2889 /**
2890 * MapR - get Zookeeper connect string for the default cluster.
2891 */
2892 public String getZkConnectString() throws IOException {
2893 throw new UnsupportedOperationException("See concrete FS for implementation");
2894 }
2895
2896 /**
2897 * MapR - get jobTracker addresses given by cluster name
2898 * in mapred.job.tracker
2899 */
2900 public InetSocketAddress[] getJobTrackerAddrs(Configuration conf)
2901 throws IOException {
2902 throw new UnsupportedOperationException("See concrete FS for implementation");
2903 }
2904 }