001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019 package org.apache.hadoop.fs;
020
021 import java.io.BufferedOutputStream;
022 import java.io.DataOutput;
023 import java.io.File;
024 import java.io.FileInputStream;
025 import java.io.FileNotFoundException;
026 import java.io.FileOutputStream;
027 import java.io.IOException;
028 import java.io.OutputStream;
029 import java.io.FileDescriptor;
030 import java.net.URI;
031 import java.nio.ByteBuffer;
032 import java.util.Arrays;
033 import java.util.EnumSet;
034 import java.util.StringTokenizer;
035
036 import org.apache.hadoop.classification.InterfaceAudience;
037 import org.apache.hadoop.classification.InterfaceStability;
038 import org.apache.hadoop.conf.Configuration;
039 import org.apache.hadoop.fs.permission.FsPermission;
040 import org.apache.hadoop.io.nativeio.NativeIO;
041 import org.apache.hadoop.io.nativeio.NativeIO.POSIX.Stat;
042 import org.apache.hadoop.util.Progressable;
043 import org.apache.hadoop.util.Shell;
044 import org.apache.hadoop.util.StringUtils;
045
046 /****************************************************************
047 * Implement the FileSystem API for the raw local filesystem.
048 *
049 *****************************************************************/
050 @InterfaceAudience.Public
051 @InterfaceStability.Stable
052 public class RawLocalFileSystem extends FileSystem {
053 static final URI NAME = URI.create("file:///");
054 private Path workingDir;
055
056 public RawLocalFileSystem() {
057 workingDir = getInitialWorkingDirectory();
058 }
059
060 private Path makeAbsolute(Path f) {
061 if (f.isAbsolute()) {
062 return f;
063 } else {
064 return new Path(workingDir, f);
065 }
066 }
067
068 /** Convert a path to a File. */
069 public File pathToFile(Path path) {
070 checkPath(path);
071 if (!path.isAbsolute()) {
072 path = new Path(getWorkingDirectory(), path);
073 }
074 return new File(path.toUri().getPath());
075 }
076
077 @Override
078 public URI getUri() { return NAME; }
079
080 @Override
081 public void initialize(URI uri, Configuration conf) throws IOException {
082 super.initialize(uri, conf);
083 setConf(conf);
084 }
085
086 /*******************************************************
087 * For open()'s FSInputStream.
088 *******************************************************/
089 class LocalFSFileInputStream extends FSInputStream implements HasFileDescriptor {
090 private FileInputStream fis;
091 private long position;
092
093 public LocalFSFileInputStream(Path f) throws IOException {
094 fis = new FileInputStream(pathToFile(f));
095 }
096
097 @Override
098 public void seek(long pos) throws IOException {
099 fis.getChannel().position(pos);
100 this.position = pos;
101 }
102
103 @Override
104 public long getPos() throws IOException {
105 return this.position;
106 }
107
108 @Override
109 public boolean seekToNewSource(long targetPos) throws IOException {
110 return false;
111 }
112
113 /*
114 * Just forward to the fis
115 */
116 @Override
117 public int available() throws IOException { return fis.available(); }
118 @Override
119 public void close() throws IOException { fis.close(); }
120 @Override
121 public boolean markSupported() { return false; }
122
123 @Override
124 public int read() throws IOException {
125 try {
126 int value = fis.read();
127 if (value >= 0) {
128 this.position++;
129 statistics.incrementBytesRead(1);
130 }
131 return value;
132 } catch (IOException e) { // unexpected exception
133 throw new FSError(e); // assume native fs error
134 }
135 }
136
137 @Override
138 public int read(byte[] b, int off, int len) throws IOException {
139 try {
140 int value = fis.read(b, off, len);
141 if (value > 0) {
142 this.position += value;
143 statistics.incrementBytesRead(value);
144 }
145 return value;
146 } catch (IOException e) { // unexpected exception
147 throw new FSError(e); // assume native fs error
148 }
149 }
150
151 @Override
152 public int read(long position, byte[] b, int off, int len)
153 throws IOException {
154 ByteBuffer bb = ByteBuffer.wrap(b, off, len);
155 try {
156 int value = fis.getChannel().read(bb, position);
157 if (value > 0) {
158 statistics.incrementBytesRead(value);
159 }
160 return value;
161 } catch (IOException e) {
162 throw new FSError(e);
163 }
164 }
165
166 @Override
167 public long skip(long n) throws IOException {
168 long value = fis.skip(n);
169 if (value > 0) {
170 this.position += value;
171 }
172 return value;
173 }
174
175 @Override
176 public FileDescriptor getFileDescriptor() throws IOException {
177 return fis.getFD();
178 }
179 }
180
181 @Override
182 public FSDataInputStream open(Path f, int bufferSize) throws IOException {
183 if (!exists(f)) {
184 throw new FileNotFoundException(f.toString());
185 }
186 return new FSDataInputStream(new BufferedFSInputStream(
187 new LocalFSFileInputStream(f), bufferSize));
188 }
189
190 /*********************************************************
191 * For create()'s FSOutputStream.
192 *********************************************************/
193 class LocalFSFileOutputStream extends OutputStream {
194 private FileOutputStream fos;
195
196 private LocalFSFileOutputStream(Path f, boolean append) throws IOException {
197 this.fos = new FileOutputStream(pathToFile(f), append);
198 }
199
200 /*
201 * Just forward to the fos
202 */
203 @Override
204 public void close() throws IOException { fos.close(); }
205 @Override
206 public void flush() throws IOException { fos.flush(); }
207 @Override
208 public void write(byte[] b, int off, int len) throws IOException {
209 try {
210 fos.write(b, off, len);
211 } catch (IOException e) { // unexpected exception
212 throw new FSError(e); // assume native fs error
213 }
214 }
215
216 @Override
217 public void write(int b) throws IOException {
218 try {
219 fos.write(b);
220 } catch (IOException e) { // unexpected exception
221 throw new FSError(e); // assume native fs error
222 }
223 }
224 }
225
226 @Override
227 public FSDataOutputStream append(Path f, int bufferSize,
228 Progressable progress) throws IOException {
229 if (!exists(f)) {
230 throw new FileNotFoundException("File " + f + " not found");
231 }
232 if (getFileStatus(f).isDirectory()) {
233 throw new IOException("Cannot append to a diretory (=" + f + " )");
234 }
235 return new FSDataOutputStream(new BufferedOutputStream(
236 new LocalFSFileOutputStream(f, true), bufferSize), statistics);
237 }
238
239 @Override
240 public FSDataOutputStream create(Path f, boolean overwrite, int bufferSize,
241 short replication, long blockSize, Progressable progress)
242 throws IOException {
243 return create(f, overwrite, true, bufferSize, replication, blockSize, progress);
244 }
245
246 private FSDataOutputStream create(Path f, boolean overwrite,
247 boolean createParent, int bufferSize, short replication, long blockSize,
248 Progressable progress) throws IOException {
249 if (exists(f) && !overwrite) {
250 throw new IOException("File already exists: "+f);
251 }
252 Path parent = f.getParent();
253 if (parent != null && !mkdirs(parent)) {
254 throw new IOException("Mkdirs failed to create " + parent.toString());
255 }
256 return new FSDataOutputStream(new BufferedOutputStream(
257 new LocalFSFileOutputStream(f, false), bufferSize), statistics);
258 }
259
260 @Override
261 @Deprecated
262 public FSDataOutputStream createNonRecursive(Path f, FsPermission permission,
263 EnumSet<CreateFlag> flags, int bufferSize, short replication, long blockSize,
264 Progressable progress) throws IOException {
265 if (exists(f) && !flags.contains(CreateFlag.OVERWRITE)) {
266 throw new IOException("File already exists: "+f);
267 }
268 return new FSDataOutputStream(new BufferedOutputStream(
269 new LocalFSFileOutputStream(f, false), bufferSize), statistics);
270 }
271
272 @Override
273 public FSDataOutputStream create(Path f, FsPermission permission,
274 boolean overwrite, int bufferSize, short replication, long blockSize,
275 Progressable progress) throws IOException {
276
277 FSDataOutputStream out = create(f,
278 overwrite, bufferSize, replication, blockSize, progress);
279 setPermission(f, permission);
280 return out;
281 }
282
283 @Override
284 public FSDataOutputStream createNonRecursive(Path f, FsPermission permission,
285 boolean overwrite,
286 int bufferSize, short replication, long blockSize,
287 Progressable progress) throws IOException {
288 FSDataOutputStream out = create(f,
289 overwrite, false, bufferSize, replication, blockSize, progress);
290 setPermission(f, permission);
291 return out;
292 }
293
294 @Override
295 public boolean rename(Path src, Path dst) throws IOException {
296 // Attempt rename using Java API.
297 File srcFile = pathToFile(src);
298 File dstFile = pathToFile(dst);
299 if (srcFile.renameTo(dstFile)) {
300 return true;
301 }
302
303 // Enforce POSIX rename behavior that a source directory replaces an existing
304 // destination if the destination is an empty directory. On most platforms,
305 // this is already handled by the Java API call above. Some platforms
306 // (notably Windows) do not provide this behavior, so the Java API call above
307 // fails. Delete destination and attempt rename again.
308 if (this.exists(dst)) {
309 FileStatus sdst = this.getFileStatus(dst);
310 if (sdst.isDirectory() && dstFile.list().length == 0) {
311 if (LOG.isDebugEnabled()) {
312 LOG.debug("Deleting empty destination and renaming " + src + " to " +
313 dst);
314 }
315 if (this.delete(dst, false) && srcFile.renameTo(dstFile)) {
316 return true;
317 }
318 }
319 }
320
321 // The fallback behavior accomplishes the rename by a full copy.
322 if (LOG.isDebugEnabled()) {
323 LOG.debug("Falling through to a copy of " + src + " to " + dst);
324 }
325 return FileUtil.copy(this, src, this, dst, true, getConf());
326 }
327
328 /**
329 * Delete the given path to a file or directory.
330 * @param p the path to delete
331 * @param recursive to delete sub-directories
332 * @return true if the file or directory and all its contents were deleted
333 * @throws IOException if p is non-empty and recursive is false
334 */
335 @Override
336 public boolean delete(Path p, boolean recursive) throws IOException {
337 File f = pathToFile(p);
338 if (f.isFile()) {
339 return f.delete();
340 } else if (!recursive && f.isDirectory() &&
341 (FileUtil.listFiles(f).length != 0)) {
342 throw new IOException("Directory " + f.toString() + " is not empty");
343 }
344 return FileUtil.fullyDelete(f);
345 }
346
347 @Override
348 public FileStatus[] listStatus(Path f) throws IOException {
349 File localf = pathToFile(f);
350 FileStatus[] results;
351
352 if (!localf.exists()) {
353 throw new FileNotFoundException("File " + f + " does not exist");
354 }
355 if (localf.isFile()) {
356 return new FileStatus[] {
357 new RawLocalFileStatus(localf, getDefaultBlockSize(f), this) };
358 }
359
360 String[] names = localf.list();
361 if (names == null) {
362 throw new FileNotFoundException(f + ": null file list");
363 }
364 results = new FileStatus[names.length];
365 int j = 0;
366 for (int i = 0; i < names.length; i++) {
367 try {
368 // Assemble the path using the Path 3 arg constructor to make sure
369 // paths with colon are properly resolved on Linux
370 results[j] = getFileStatus(new Path(f, new Path(null, null, names[i])));
371 j++;
372 } catch (FileNotFoundException e) {
373 // ignore the files not found since the dir list may have have changed
374 // since the names[] list was generated.
375 }
376 }
377 if (j == names.length) {
378 return results;
379 }
380 return Arrays.copyOf(results, j);
381 }
382
383 /**
384 * Creates the specified directory hierarchy. Does not
385 * treat existence as an error.
386 */
387 @Override
388 public boolean mkdirs(Path f) throws IOException {
389 if(f == null) {
390 throw new IllegalArgumentException("mkdirs path arg is null");
391 }
392 Path parent = f.getParent();
393 File p2f = pathToFile(f);
394 if(parent != null) {
395 File parent2f = pathToFile(parent);
396 if(parent2f != null && parent2f.exists() && !parent2f.isDirectory()) {
397 throw new FileAlreadyExistsException("Parent path is not a directory: "
398 + parent);
399 }
400 }
401 return (parent == null || mkdirs(parent)) &&
402 (p2f.mkdir() || p2f.isDirectory());
403 }
404
405 @Override
406 public boolean mkdirs(Path f, FsPermission permission) throws IOException {
407 boolean b = mkdirs(f);
408 if(b) {
409 setPermission(f, permission);
410 }
411 return b;
412 }
413
414
415 @Override
416 protected boolean primitiveMkdir(Path f, FsPermission absolutePermission)
417 throws IOException {
418 boolean b = mkdirs(f);
419 setPermission(f, absolutePermission);
420 return b;
421 }
422
423
424 @Override
425 public Path getHomeDirectory() {
426 return this.makeQualified(new Path(System.getProperty("user.home")));
427 }
428
429 /**
430 * Set the working directory to the given directory.
431 */
432 @Override
433 public void setWorkingDirectory(Path newDir) {
434 workingDir = makeAbsolute(newDir);
435 checkPath(workingDir);
436
437 }
438
439 @Override
440 public Path getWorkingDirectory() {
441 return workingDir;
442 }
443
444 @Override
445 protected Path getInitialWorkingDirectory() {
446 return this.makeQualified(new Path(System.getProperty("user.dir")));
447 }
448
449 @Override
450 public FsStatus getStatus(Path p) throws IOException {
451 File partition = pathToFile(p == null ? new Path("/") : p);
452 //File provides getUsableSpace() and getFreeSpace()
453 //File provides no API to obtain used space, assume used = total - free
454 return new FsStatus(partition.getTotalSpace(),
455 partition.getTotalSpace() - partition.getFreeSpace(),
456 partition.getFreeSpace());
457 }
458
459 // In the case of the local filesystem, we can just rename the file.
460 @Override
461 public void moveFromLocalFile(Path src, Path dst) throws IOException {
462 rename(src, dst);
463 }
464
465 // We can write output directly to the final location
466 @Override
467 public Path startLocalOutput(Path fsOutputFile, Path tmpLocalFile)
468 throws IOException {
469 return fsOutputFile;
470 }
471
472 // It's in the right place - nothing to do.
473 @Override
474 public void completeLocalOutput(Path fsWorkingFile, Path tmpLocalFile)
475 throws IOException {
476 }
477
478 @Override
479 public void close() throws IOException {
480 super.close();
481 }
482
483 @Override
484 public String toString() {
485 return "LocalFS";
486 }
487
488 @Override
489 public FileStatus getFileStatus(Path f) throws IOException {
490 File path = pathToFile(f);
491 if (path.exists()) {
492 return new RawLocalFileStatus(pathToFile(f), getDefaultBlockSize(f), this);
493 } else {
494 throw new FileNotFoundException("File " + f + " does not exist");
495 }
496 }
497
498 static class RawLocalFileStatus extends FileStatus {
499 /* We can add extra fields here. It breaks at least CopyFiles.FilePair().
500 * We recognize if the information is already loaded by check if
501 * onwer.equals("").
502 */
503 private boolean isPermissionLoaded() {
504 return !super.getOwner().isEmpty();
505 }
506
507 RawLocalFileStatus(File f, long defaultBlockSize, FileSystem fs) {
508 super(f.length(), f.isDirectory(), 1, defaultBlockSize,
509 f.lastModified(), new Path(f.getPath()).makeQualified(fs.getUri(),
510 fs.getWorkingDirectory()));
511 }
512
513 @Override
514 public FsPermission getPermission() {
515 if (!isPermissionLoaded()) {
516 loadPermissionInfo();
517 }
518 return super.getPermission();
519 }
520
521 @Override
522 public String getOwner() {
523 if (!isPermissionLoaded()) {
524 loadPermissionInfo();
525 }
526 return super.getOwner();
527 }
528
529 @Override
530 public String getGroup() {
531 if (!isPermissionLoaded()) {
532 loadPermissionInfo();
533 }
534 return super.getGroup();
535 }
536
537 /// loads permissions, owner, and group from `ls -ld`
538 private void loadPermissionInfo() {
539 if (NativeIO.isAvailable()) {
540 loadNativePermissionInfo();
541 } else {
542 IOException e = null;
543 try {
544 String output = FileUtil.execCommand(new File(getPath().toUri()),
545 Shell.getGetPermissionCommand());
546 StringTokenizer t =
547 new StringTokenizer(output, Shell.TOKEN_SEPARATOR_REGEX);
548 //expected format
549 //-rw------- 1 username groupname ...
550 String permission = t.nextToken();
551 if (permission.length() > 10) { //files with ACLs might have a '+'
552 permission = permission.substring(0, 10);
553 }
554 setPermission(FsPermission.valueOf(permission));
555 t.nextToken();
556
557 String owner = t.nextToken();
558 // If on windows domain, token format is DOMAIN\\user and we want to
559 // extract only the user name
560 if (Shell.WINDOWS) {
561 int i = owner.indexOf('\\');
562 if (i != -1)
563 owner = owner.substring(i + 1);
564 }
565 setOwner(owner);
566
567 setGroup(t.nextToken());
568 } catch (Shell.ExitCodeException ioe) {
569 if (ioe.getExitCode() != 1) {
570 e = ioe;
571 } else {
572 setPermission(null);
573 setOwner(null);
574 setGroup(null);
575 }
576 } catch (IOException ioe) {
577 e = ioe;
578 } finally {
579 if (e != null) {
580 throw new RuntimeException("Error while running command to get " +
581 "file permissions : " +
582 StringUtils.stringifyException(e));
583 }
584 }
585 }
586 }
587
588 private void loadNativePermissionInfo() {
589 if (NativeIO.isAvailable()) {
590 final String pstr = getPath().toUri().getPath();
591 FileDescriptor fd;
592 try {
593 fd = NativeIO.POSIX.open(pstr, NativeIO.POSIX.O_RDONLY, 0);
594 } catch (IOException maybeDir) {
595 try {
596 fd = NativeIO.POSIX.open(pstr, NativeIO.POSIX.O_DIRECTORY, 0);
597 } catch (IOException e) {
598 if (LOG.isErrorEnabled()) {
599 LOG.error("Failed to fstat on: " + pstr, e);
600 }
601 throw new RuntimeException(pstr, e);
602 }
603 }
604
605 final FileInputStream dummy = new FileInputStream(fd);
606 try {
607 final Stat st = NativeIO.POSIX.getFstat(fd);
608 setPermission(new FsPermission((short)st.getMode()));
609 setOwner(st.getOwner());
610 setGroup(st.getGroup());
611 } catch (IOException e) {
612 if (LOG.isInfoEnabled()) {
613 LOG.info("Native fstat failed.", e);
614 }
615 } finally {
616 try {
617 dummy.close();
618 } catch (IOException e) {
619 if (LOG.isInfoEnabled()) {
620 LOG.info("Failed to close dummy descriptor.", e);
621 }
622 }
623 }
624 }
625 }
626
627 @Override
628 public void write(DataOutput out) throws IOException {
629 if (!isPermissionLoaded()) {
630 loadPermissionInfo();
631 }
632 super.write(out);
633 }
634 }
635
636 /**
637 * Use the command chown to set owner.
638 */
639 @Override
640 public void setOwner(Path p, String username, String groupname)
641 throws IOException {
642 FileUtil.setOwner(pathToFile(p), username, groupname);
643 }
644
645 /**
646 * Use the command chmod to set permission.
647 */
648 @Override
649 public void setPermission(Path p, FsPermission permission)
650 throws IOException {
651 if (NativeIO.isAvailable()) {
652 NativeIO.POSIX.chmod(pathToFile(p).getCanonicalPath(),
653 permission.toShort());
654 } else {
655 String perm = String.format("%05o", permission.toShort());
656 Shell.execCommand(Shell.getSetPermissionCommand(perm, false,
657 FileUtil.makeShellPath(pathToFile(p), true)));
658 }
659 }
660
661 /**
662 * Sets the {@link Path}'s last modified time <em>only</em> to the given
663 * valid time.
664 *
665 * @param mtime the modification time to set (only if greater than zero).
666 * @param atime currently ignored.
667 * @throws IOException if setting the last modified time fails.
668 */
669 @Override
670 public void setTimes(Path p, long mtime, long atime) throws IOException {
671 File f = pathToFile(p);
672 if(mtime >= 0) {
673 if(!f.setLastModified(mtime)) {
674 throw new IOException(
675 "couldn't set last-modified time to " +
676 mtime +
677 " for " +
678 f.getAbsolutePath());
679 }
680 }
681 }
682
683 @Override
684 public void createSymlink(Path oldPath, Path newPath, boolean createParent)
685 throws IOException {
686
687 if (oldPath == null) {
688 throw new RuntimeException("Target cannot be null");
689 }
690
691 if (newPath == null) {
692 throw new RuntimeException("Link cannot be null");
693 }
694
695 final String target = pathToFile(oldPath).toString();
696 final String link = pathToFile(newPath).toString();
697
698 if (NativeIO.isAvailable()) {
699 if (createParent) {
700 // Create any missing ancestors
701 Path parentOfNewPath = newPath.getParent();
702 if (parentOfNewPath != null) {
703 if (!mkdirs(parentOfNewPath)) {
704 throw new IOException("Mkdirs failed " + parentOfNewPath.toString());
705 }
706 }
707 }
708
709 if (LOG.isDebugEnabled()) {
710 LOG.debug("NativeIO.symlink: " + target + " <- " + link);
711 }
712 NativeIO.POSIX.symlink(target, link);
713 } else {
714 FileUtil.symLink(target, link);
715 }
716 }
717
718 }