001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019 package org.apache.hadoop.fs;
020
021 import java.io.BufferedInputStream;
022 import java.io.BufferedOutputStream;
023 import java.io.File;
024 import java.io.FileInputStream;
025 import java.io.FileOutputStream;
026 import java.io.IOException;
027 import java.io.InputStream;
028 import java.io.OutputStream;
029 import java.util.ArrayList;
030 import java.util.Arrays;
031 import java.util.Enumeration;
032 import java.util.List;
033 import java.util.Map;
034 import java.util.jar.Attributes;
035 import java.util.jar.JarOutputStream;
036 import java.util.jar.Manifest;
037 import java.util.zip.GZIPInputStream;
038 import java.util.zip.ZipEntry;
039 import java.util.zip.ZipFile;
040
041 import org.apache.commons.collections.map.CaseInsensitiveMap;
042 import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
043 import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
044 import org.apache.commons.logging.Log;
045 import org.apache.commons.logging.LogFactory;
046 import org.apache.hadoop.classification.InterfaceAudience;
047 import org.apache.hadoop.classification.InterfaceStability;
048 import org.apache.hadoop.conf.Configuration;
049 import org.apache.hadoop.fs.permission.ChmodParser;
050 import org.apache.hadoop.fs.permission.FsAction;
051 import org.apache.hadoop.fs.permission.FsPermission;
052 import org.apache.hadoop.io.IOUtils;
053 import org.apache.hadoop.io.nativeio.NativeIO;
054 import org.apache.hadoop.util.Shell;
055 import org.apache.hadoop.util.Shell.ShellCommandExecutor;
056 import org.apache.hadoop.util.StringUtils;
057
058 /**
059 * A collection of file-processing util methods
060 */
061 @InterfaceAudience.Public
062 @InterfaceStability.Evolving
063 public class FileUtil {
064
065 private static final Log LOG = LogFactory.getLog(FileUtil.class);
066
067 /* The error code is defined in winutils to indicate insufficient
068 * privilege to create symbolic links. This value need to keep in
069 * sync with the constant of the same name in:
070 * "src\winutils\common.h"
071 * */
072 public static final int SYMLINK_NO_PRIVILEGE = 2;
073
074 /**
075 * convert an array of FileStatus to an array of Path
076 *
077 * @param stats
078 * an array of FileStatus objects
079 * @return an array of paths corresponding to the input
080 */
081 public static Path[] stat2Paths(FileStatus[] stats) {
082 if (stats == null)
083 return null;
084 Path[] ret = new Path[stats.length];
085 for (int i = 0; i < stats.length; ++i) {
086 ret[i] = stats[i].getPath();
087 }
088 return ret;
089 }
090
091 /**
092 * convert an array of FileStatus to an array of Path.
093 * If stats if null, return path
094 * @param stats
095 * an array of FileStatus objects
096 * @param path
097 * default path to return in stats is null
098 * @return an array of paths corresponding to the input
099 */
100 public static Path[] stat2Paths(FileStatus[] stats, Path path) {
101 if (stats == null)
102 return new Path[]{path};
103 else
104 return stat2Paths(stats);
105 }
106
107 /**
108 * Delete a directory and all its contents. If
109 * we return false, the directory may be partially-deleted.
110 * (1) If dir is symlink to a file, the symlink is deleted. The file pointed
111 * to by the symlink is not deleted.
112 * (2) If dir is symlink to a directory, symlink is deleted. The directory
113 * pointed to by symlink is not deleted.
114 * (3) If dir is a normal file, it is deleted.
115 * (4) If dir is a normal directory, then dir and all its contents recursively
116 * are deleted.
117 */
118 public static boolean fullyDelete(final File dir) {
119 return fullyDelete(dir, false);
120 }
121
122 /**
123 * Delete a directory and all its contents. If
124 * we return false, the directory may be partially-deleted.
125 * (1) If dir is symlink to a file, the symlink is deleted. The file pointed
126 * to by the symlink is not deleted.
127 * (2) If dir is symlink to a directory, symlink is deleted. The directory
128 * pointed to by symlink is not deleted.
129 * (3) If dir is a normal file, it is deleted.
130 * (4) If dir is a normal directory, then dir and all its contents recursively
131 * are deleted.
132 * @param dir the file or directory to be deleted
133 * @param tryGrantPermissions true if permissions should be modified to delete a file.
134 * @return true on success false on failure.
135 */
136 public static boolean fullyDelete(final File dir, boolean tryGrantPermissions) {
137 if (tryGrantPermissions) {
138 // try to chmod +rwx the parent folder of the 'dir':
139 File parent = dir.getParentFile();
140 grantPermissions(parent);
141 }
142 if (deleteImpl(dir, false)) {
143 // dir is (a) normal file, (b) symlink to a file, (c) empty directory or
144 // (d) symlink to a directory
145 return true;
146 }
147 // handle nonempty directory deletion
148 if (!fullyDeleteContents(dir, tryGrantPermissions)) {
149 return false;
150 }
151 return deleteImpl(dir, true);
152 }
153
154 /**
155 * Returns the target of the given symlink. Returns the empty string if
156 * the given path does not refer to a symlink or there is an error
157 * accessing the symlink.
158 * @param f File representing the symbolic link.
159 * @return The target of the symbolic link, empty string on error or if not
160 * a symlink.
161 */
162 public static String readLink(File f) {
163 /* NB: Use readSymbolicLink in java.nio.file.Path once available. Could
164 * use getCanonicalPath in File to get the target of the symlink but that
165 * does not indicate if the given path refers to a symlink.
166 */
167 try {
168 return Shell.execCommand(
169 Shell.getReadlinkCommand(f.toString())).trim();
170 } catch (IOException x) {
171 return "";
172 }
173 }
174
175 /*
176 * Pure-Java implementation of "chmod +rwx f".
177 */
178 private static void grantPermissions(final File f) {
179 FileUtil.setExecutable(f, true);
180 FileUtil.setReadable(f, true);
181 FileUtil.setWritable(f, true);
182 }
183
184 private static boolean deleteImpl(final File f, final boolean doLog) {
185 if (f == null) {
186 LOG.warn("null file argument.");
187 return false;
188 }
189 final boolean wasDeleted = f.delete();
190 if (wasDeleted) {
191 return true;
192 }
193 final boolean ex = f.exists();
194 if (doLog && ex) {
195 LOG.warn("Failed to delete file or dir ["
196 + f.getAbsolutePath() + "]: it still exists.");
197 }
198 return !ex;
199 }
200
201 /**
202 * Delete the contents of a directory, not the directory itself. If
203 * we return false, the directory may be partially-deleted.
204 * If dir is a symlink to a directory, all the contents of the actual
205 * directory pointed to by dir will be deleted.
206 */
207 public static boolean fullyDeleteContents(final File dir) {
208 return fullyDeleteContents(dir, false);
209 }
210
211 /**
212 * Delete the contents of a directory, not the directory itself. If
213 * we return false, the directory may be partially-deleted.
214 * If dir is a symlink to a directory, all the contents of the actual
215 * directory pointed to by dir will be deleted.
216 * @param tryGrantPermissions if 'true', try grant +rwx permissions to this
217 * and all the underlying directories before trying to delete their contents.
218 */
219 public static boolean fullyDeleteContents(final File dir, final boolean tryGrantPermissions) {
220 if (tryGrantPermissions) {
221 // to be able to list the dir and delete files from it
222 // we must grant the dir rwx permissions:
223 grantPermissions(dir);
224 }
225 boolean deletionSucceeded = true;
226 final File[] contents = dir.listFiles();
227 if (contents != null) {
228 for (int i = 0; i < contents.length; i++) {
229 if (contents[i].isFile()) {
230 if (!deleteImpl(contents[i], true)) {// normal file or symlink to another file
231 deletionSucceeded = false;
232 continue; // continue deletion of other files/dirs under dir
233 }
234 } else {
235 // Either directory or symlink to another directory.
236 // Try deleting the directory as this might be a symlink
237 boolean b = false;
238 b = deleteImpl(contents[i], false);
239 if (b){
240 //this was indeed a symlink or an empty directory
241 continue;
242 }
243 // if not an empty directory or symlink let
244 // fullydelete handle it.
245 if (!fullyDelete(contents[i], tryGrantPermissions)) {
246 deletionSucceeded = false;
247 // continue deletion of other files/dirs under dir
248 }
249 }
250 }
251 }
252 return deletionSucceeded;
253 }
254
255 /**
256 * Recursively delete a directory.
257 *
258 * @param fs {@link FileSystem} on which the path is present
259 * @param dir directory to recursively delete
260 * @throws IOException
261 * @deprecated Use {@link FileSystem#delete(Path, boolean)}
262 */
263 @Deprecated
264 public static void fullyDelete(FileSystem fs, Path dir)
265 throws IOException {
266 fs.delete(dir, true);
267 }
268
269 //
270 // If the destination is a subdirectory of the source, then
271 // generate exception
272 //
273 private static void checkDependencies(FileSystem srcFS,
274 Path src,
275 FileSystem dstFS,
276 Path dst)
277 throws IOException {
278 if (srcFS == dstFS) {
279 String srcq = src.makeQualified(srcFS).toString() + Path.SEPARATOR;
280 String dstq = dst.makeQualified(dstFS).toString() + Path.SEPARATOR;
281 if (dstq.startsWith(srcq)) {
282 if (srcq.length() == dstq.length()) {
283 throw new IOException("Cannot copy " + src + " to itself.");
284 } else {
285 throw new IOException("Cannot copy " + src + " to its subdirectory " +
286 dst);
287 }
288 }
289 }
290 }
291
292 /** Copy files between FileSystems. */
293 public static boolean copy(FileSystem srcFS, Path src,
294 FileSystem dstFS, Path dst,
295 boolean deleteSource,
296 Configuration conf) throws IOException {
297 return copy(srcFS, src, dstFS, dst, deleteSource, true, conf);
298 }
299
300 public static boolean copy(FileSystem srcFS, Path[] srcs,
301 FileSystem dstFS, Path dst,
302 boolean deleteSource,
303 boolean overwrite, Configuration conf)
304 throws IOException {
305 boolean gotException = false;
306 boolean returnVal = true;
307 StringBuilder exceptions = new StringBuilder();
308
309 if (srcs.length == 1)
310 return copy(srcFS, srcs[0], dstFS, dst, deleteSource, overwrite, conf);
311
312 // Check if dest is directory
313 if (!dstFS.exists(dst)) {
314 throw new IOException("`" + dst +"': specified destination directory " +
315 "does not exist");
316 } else {
317 FileStatus sdst = dstFS.getFileStatus(dst);
318 if (!sdst.isDirectory())
319 throw new IOException("copying multiple files, but last argument `" +
320 dst + "' is not a directory");
321 }
322
323 for (Path src : srcs) {
324 try {
325 if (!copy(srcFS, src, dstFS, dst, deleteSource, overwrite, conf))
326 returnVal = false;
327 } catch (IOException e) {
328 gotException = true;
329 exceptions.append(e.getMessage());
330 exceptions.append("\n");
331 }
332 }
333 if (gotException) {
334 throw new IOException(exceptions.toString());
335 }
336 return returnVal;
337 }
338
339 /** Copy files between FileSystems. */
340 public static boolean copy(FileSystem srcFS, Path src,
341 FileSystem dstFS, Path dst,
342 boolean deleteSource,
343 boolean overwrite,
344 Configuration conf) throws IOException {
345 FileStatus fileStatus = srcFS.getFileStatus(src);
346 return copy(srcFS, fileStatus, dstFS, dst, deleteSource, overwrite, conf);
347 }
348
349 /** Copy files between FileSystems. */
350 private static boolean copy(FileSystem srcFS, FileStatus srcStatus,
351 FileSystem dstFS, Path dst,
352 boolean deleteSource,
353 boolean overwrite,
354 Configuration conf) throws IOException {
355 Path src = srcStatus.getPath();
356 dst = checkDest(src.getName(), dstFS, dst, overwrite);
357 if (srcStatus.isDirectory()) {
358 checkDependencies(srcFS, src, dstFS, dst);
359 if (!dstFS.mkdirs(dst)) {
360 return false;
361 }
362 FileStatus contents[] = srcFS.listStatus(src);
363 for (int i = 0; i < contents.length; i++) {
364 copy(srcFS, contents[i], dstFS,
365 new Path(dst, contents[i].getPath().getName()),
366 deleteSource, overwrite, conf);
367 }
368 } else if (srcStatus.isTable()) {
369 throw new IOException("Cannot copy MDP Tables");
370 } else {
371 InputStream in=null;
372 OutputStream out = null;
373 try {
374 in = srcFS.open(src);
375 out = dstFS.create(dst, overwrite);
376 IOUtils.copyBytes(in, out, conf, true);
377 } catch (IOException e) {
378 IOUtils.closeStream(out);
379 IOUtils.closeStream(in);
380 throw e;
381 }
382 }
383 if (deleteSource) {
384 return srcFS.delete(src, true);
385 } else {
386 return true;
387 }
388
389 }
390
391 /** Copy all files in a directory to one output file (merge). */
392 public static boolean copyMerge(FileSystem srcFS, Path srcDir,
393 FileSystem dstFS, Path dstFile,
394 boolean deleteSource,
395 Configuration conf, String addString) throws IOException {
396 dstFile = checkDest(srcDir.getName(), dstFS, dstFile, false);
397
398 if (!srcFS.getFileStatus(srcDir).isDirectory())
399 return false;
400
401 OutputStream out = dstFS.create(dstFile);
402
403 try {
404 FileStatus contents[] = srcFS.listStatus(srcDir);
405 Arrays.sort(contents);
406 for (int i = 0; i < contents.length; i++) {
407 if (contents[i].isFile()) {
408 InputStream in = srcFS.open(contents[i].getPath());
409 try {
410 IOUtils.copyBytes(in, out, conf, false);
411 if (addString!=null)
412 out.write(addString.getBytes("UTF-8"));
413
414 } finally {
415 in.close();
416 }
417 }
418 }
419 } finally {
420 out.close();
421 }
422
423
424 if (deleteSource) {
425 return srcFS.delete(srcDir, true);
426 } else {
427 return true;
428 }
429 }
430
431 /** Copy local files to a FileSystem. */
432 public static boolean copy(File src,
433 FileSystem dstFS, Path dst,
434 boolean deleteSource,
435 Configuration conf) throws IOException {
436 dst = checkDest(src.getName(), dstFS, dst, false);
437
438 if (src.isDirectory()) {
439 if (!dstFS.mkdirs(dst)) {
440 return false;
441 }
442 File contents[] = listFiles(src);
443 for (int i = 0; i < contents.length; i++) {
444 copy(contents[i], dstFS, new Path(dst, contents[i].getName()),
445 deleteSource, conf);
446 }
447 } else if (src.isFile()) {
448 InputStream in = null;
449 OutputStream out =null;
450 try {
451 in = new FileInputStream(src);
452 out = dstFS.create(dst);
453 IOUtils.copyBytes(in, out, conf);
454 } catch (IOException e) {
455 IOUtils.closeStream( out );
456 IOUtils.closeStream( in );
457 throw e;
458 }
459 } else {
460 throw new IOException(src.toString() +
461 ": No such file or directory");
462 }
463 if (deleteSource) {
464 return FileUtil.fullyDelete(src);
465 } else {
466 return true;
467 }
468 }
469
470 /** Copy FileSystem files to local files. */
471 public static boolean copy(FileSystem srcFS, Path src,
472 File dst, boolean deleteSource,
473 Configuration conf) throws IOException {
474 FileStatus filestatus = srcFS.getFileStatus(src);
475 return copy(srcFS, filestatus, dst, deleteSource, conf);
476 }
477
478 /** Copy FileSystem files to local files. */
479 private static boolean copy(FileSystem srcFS, FileStatus srcStatus,
480 File dst, boolean deleteSource,
481 Configuration conf) throws IOException {
482 Path src = srcStatus.getPath();
483 if (srcStatus.isDirectory()) {
484 if (!dst.mkdirs()) {
485 return false;
486 }
487 FileStatus contents[] = srcFS.listStatus(src);
488 for (int i = 0; i < contents.length; i++) {
489 copy(srcFS, contents[i],
490 new File(dst, contents[i].getPath().getName()),
491 deleteSource, conf);
492 }
493 } else {
494 InputStream in = srcFS.open(src);
495 IOUtils.copyBytes(in, new FileOutputStream(dst), conf);
496 }
497 if (deleteSource) {
498 return srcFS.delete(src, true);
499 } else {
500 return true;
501 }
502 }
503
504 private static Path checkDest(String srcName, FileSystem dstFS, Path dst,
505 boolean overwrite) throws IOException {
506 if (dstFS.exists(dst)) {
507 FileStatus sdst = dstFS.getFileStatus(dst);
508 if (sdst.isDirectory()) {
509 if (null == srcName) {
510 throw new IOException("Target " + dst + " is a directory");
511 }
512 return checkDest(null, dstFS, new Path(dst, srcName), overwrite);
513 } else if (!overwrite) {
514 throw new IOException("Target " + dst + " already exists");
515 }
516 }
517 return dst;
518 }
519
520 /**
521 * Convert a os-native filename to a path that works for the shell.
522 * @param filename The filename to convert
523 * @return The unix pathname
524 * @throws IOException on windows, there can be problems with the subprocess
525 */
526 public static String makeShellPath(String filename) throws IOException {
527 return filename;
528 }
529
530 /**
531 * Convert a os-native filename to a path that works for the shell.
532 * @param file The filename to convert
533 * @return The unix pathname
534 * @throws IOException on windows, there can be problems with the subprocess
535 */
536 public static String makeShellPath(File file) throws IOException {
537 return makeShellPath(file, false);
538 }
539
540 /**
541 * Convert a os-native filename to a path that works for the shell.
542 * @param file The filename to convert
543 * @param makeCanonicalPath
544 * Whether to make canonical path for the file passed
545 * @return The unix pathname
546 * @throws IOException on windows, there can be problems with the subprocess
547 */
548 public static String makeShellPath(File file, boolean makeCanonicalPath)
549 throws IOException {
550 if (makeCanonicalPath) {
551 return makeShellPath(file.getCanonicalPath());
552 } else {
553 return makeShellPath(file.toString());
554 }
555 }
556
557 /**
558 * Takes an input dir and returns the du on that local directory. Very basic
559 * implementation.
560 *
561 * @param dir
562 * The input dir to get the disk space of this local dir
563 * @return The total disk space of the input local directory
564 */
565 public static long getDU(File dir) {
566 long size = 0;
567 if (!dir.exists())
568 return 0;
569 if (!dir.isDirectory()) {
570 return dir.length();
571 } else {
572 File[] allFiles = dir.listFiles();
573 if(allFiles != null) {
574 for (int i = 0; i < allFiles.length; i++) {
575 boolean isSymLink;
576 try {
577 isSymLink = org.apache.commons.io.FileUtils.isSymlink(allFiles[i]);
578 } catch(IOException ioe) {
579 isSymLink = true;
580 }
581 if(!isSymLink) {
582 size += getDU(allFiles[i]);
583 }
584 }
585 }
586 return size;
587 }
588 }
589
590 /**
591 * Given a File input it will unzip the file in a the unzip directory
592 * passed as the second parameter
593 * @param inFile The zip file as input
594 * @param unzipDir The unzip directory where to unzip the zip file.
595 * @throws IOException
596 */
597 public static void unZip(File inFile, File unzipDir) throws IOException {
598 Enumeration<? extends ZipEntry> entries;
599 ZipFile zipFile = new ZipFile(inFile);
600
601 try {
602 entries = zipFile.entries();
603 while (entries.hasMoreElements()) {
604 ZipEntry entry = entries.nextElement();
605 if (!entry.isDirectory()) {
606 InputStream in = zipFile.getInputStream(entry);
607 try {
608 File file = new File(unzipDir, entry.getName());
609 if (!file.getParentFile().mkdirs()) {
610 if (!file.getParentFile().isDirectory()) {
611 throw new IOException("Mkdirs failed to create " +
612 file.getParentFile().toString());
613 }
614 }
615 OutputStream out = new FileOutputStream(file);
616 try {
617 byte[] buffer = new byte[8192];
618 int i;
619 while ((i = in.read(buffer)) != -1) {
620 out.write(buffer, 0, i);
621 }
622 } finally {
623 out.close();
624 }
625 } finally {
626 in.close();
627 }
628 }
629 }
630 } finally {
631 zipFile.close();
632 }
633 }
634
635 /**
636 * Given a Tar File as input it will untar the file in a the untar directory
637 * passed as the second parameter
638 *
639 * This utility will untar ".tar" files and ".tar.gz","tgz" files.
640 *
641 * @param inFile The tar file as input.
642 * @param untarDir The untar directory where to untar the tar file.
643 * @throws IOException
644 */
645 public static void unTar(File inFile, File untarDir) throws IOException {
646 if (!untarDir.mkdirs()) {
647 if (!untarDir.isDirectory()) {
648 throw new IOException("Mkdirs failed to create " + untarDir);
649 }
650 }
651
652 boolean gzipped = inFile.toString().endsWith("gz");
653 if(Shell.WINDOWS) {
654 // Tar is not native to Windows. Use simple Java based implementation for
655 // tests and simple tar archives
656 unTarUsingJava(inFile, untarDir, gzipped);
657 }
658 else {
659 // spawn tar utility to untar archive for full fledged unix behavior such
660 // as resolving symlinks in tar archives
661 unTarUsingTar(inFile, untarDir, gzipped);
662 }
663 }
664
665 private static void unTarUsingTar(File inFile, File untarDir,
666 boolean gzipped) throws IOException {
667 StringBuffer untarCommand = new StringBuffer();
668 if (gzipped) {
669 untarCommand.append(" gzip -dc '");
670 untarCommand.append(FileUtil.makeShellPath(inFile));
671 untarCommand.append("' | (");
672 }
673 untarCommand.append("cd '");
674 untarCommand.append(FileUtil.makeShellPath(untarDir));
675 untarCommand.append("' ; ");
676 untarCommand.append("tar -xf ");
677
678 if (gzipped) {
679 untarCommand.append(" -)");
680 } else {
681 untarCommand.append(FileUtil.makeShellPath(inFile));
682 }
683 String[] shellCmd = { "bash", "-c", untarCommand.toString() };
684 ShellCommandExecutor shexec = new ShellCommandExecutor(shellCmd);
685 shexec.execute();
686 int exitcode = shexec.getExitCode();
687 if (exitcode != 0) {
688 throw new IOException("Error untarring file " + inFile +
689 ". Tar process exited with exit code " + exitcode);
690 }
691 }
692
693 private static void unTarUsingJava(File inFile, File untarDir,
694 boolean gzipped) throws IOException {
695 InputStream inputStream = null;
696 TarArchiveInputStream tis = null;
697 try {
698 if (gzipped) {
699 inputStream = new BufferedInputStream(new GZIPInputStream(
700 new FileInputStream(inFile)));
701 } else {
702 inputStream = new BufferedInputStream(new FileInputStream(inFile));
703 }
704
705 tis = new TarArchiveInputStream(inputStream);
706
707 for (TarArchiveEntry entry = tis.getNextTarEntry(); entry != null;) {
708 unpackEntries(tis, entry, untarDir);
709 entry = tis.getNextTarEntry();
710 }
711 } finally {
712 IOUtils.cleanup(LOG, tis, inputStream);
713 }
714 }
715
716 private static void unpackEntries(TarArchiveInputStream tis,
717 TarArchiveEntry entry, File outputDir) throws IOException {
718 if (entry.isDirectory()) {
719 File subDir = new File(outputDir, entry.getName());
720 if (!subDir.mkdir() && !subDir.isDirectory()) {
721 throw new IOException("Mkdirs failed to create tar internal dir "
722 + outputDir);
723 }
724
725 for (TarArchiveEntry e : entry.getDirectoryEntries()) {
726 unpackEntries(tis, e, subDir);
727 }
728
729 return;
730 }
731
732 File outputFile = new File(outputDir, entry.getName());
733 if (!outputDir.exists()) {
734 if (!outputDir.mkdirs()) {
735 throw new IOException("Mkdirs failed to create tar internal dir "
736 + outputDir);
737 }
738 }
739
740 int count;
741 byte data[] = new byte[2048];
742 BufferedOutputStream outputStream = new BufferedOutputStream(
743 new FileOutputStream(outputFile));
744
745 while ((count = tis.read(data)) != -1) {
746 outputStream.write(data, 0, count);
747 }
748
749 outputStream.flush();
750 outputStream.close();
751 }
752
753 /**
754 * Class for creating hardlinks.
755 * Supports Unix, WindXP.
756 * @deprecated Use {@link org.apache.hadoop.fs.HardLink}
757 */
758 @Deprecated
759 public static class HardLink extends org.apache.hadoop.fs.HardLink {
760 // This is a stub to assist with coordinated change between
761 // COMMON and HDFS projects. It will be removed after the
762 // corresponding change is committed to HDFS.
763 }
764
765 /**
766 * Create a soft link between a src and destination
767 * only on a local disk. HDFS does not support this.
768 * On Windows, when symlink creation fails due to security
769 * setting, we will log a warning. The return code in this
770 * case is 2.
771 *
772 * @param target the target for symlink
773 * @param linkname the symlink
774 * @return 0 on success
775 */
776 public static int symLink(String target, String linkname) throws IOException{
777 // Run the input paths through Java's File so that they are converted to the
778 // native OS form
779 File targetFile = new File(
780 Path.getPathWithoutSchemeAndAuthority(new Path(target)).toString());
781 File linkFile = new File(
782 Path.getPathWithoutSchemeAndAuthority(new Path(linkname)).toString());
783
784 // If not on Java7+, copy a file instead of creating a symlink since
785 // Java6 has close to no support for symlinks on Windows. Specifically
786 // File#length and File#renameTo do not work as expected.
787 // (see HADOOP-9061 for additional details)
788 // We still create symlinks for directories, since the scenario in this
789 // case is different. The directory content could change in which
790 // case the symlink loses its purpose (for example task attempt log folder
791 // is symlinked under userlogs and userlogs are generated afterwards).
792 if (Shell.WINDOWS && !Shell.isJava7OrAbove() && targetFile.isFile()) {
793 try {
794 LOG.warn("FileUtil#symlink: On Windows+Java6, copying file instead " +
795 "of creating a symlink. Copying " + target + " -> " + linkname);
796
797 if (!linkFile.getParentFile().exists()) {
798 LOG.warn("Parent directory " + linkFile.getParent() +
799 " does not exist.");
800 return 1;
801 } else {
802 org.apache.commons.io.FileUtils.copyFile(targetFile, linkFile);
803 }
804 } catch (IOException ex) {
805 LOG.warn("FileUtil#symlink failed to copy the file with error: "
806 + ex.getMessage());
807 // Exit with non-zero exit code
808 return 1;
809 }
810 return 0;
811 }
812
813 String[] cmd = Shell.getSymlinkCommand(
814 targetFile.toString(),
815 linkFile.toString());
816
817 ShellCommandExecutor shExec;
818 try {
819 if (Shell.WINDOWS &&
820 linkFile.getParentFile() != null &&
821 !new Path(target).isAbsolute()) {
822 // Relative links on Windows must be resolvable at the time of
823 // creation. To ensure this we run the shell command in the directory
824 // of the link.
825 //
826 shExec = new ShellCommandExecutor(cmd, linkFile.getParentFile());
827 } else {
828 shExec = new ShellCommandExecutor(cmd);
829 }
830 shExec.execute();
831 } catch (Shell.ExitCodeException ec) {
832 int returnVal = ec.getExitCode();
833 if (Shell.WINDOWS && returnVal == SYMLINK_NO_PRIVILEGE) {
834 LOG.warn("Fail to create symbolic links on Windows. "
835 + "The default security settings in Windows disallow non-elevated "
836 + "administrators and all non-administrators from creating symbolic links. "
837 + "This behavior can be changed in the Local Security Policy management console");
838 } else if (returnVal != 0) {
839 LOG.warn("Command '" + StringUtils.join(" ", cmd) + "' failed "
840 + returnVal + " with: " + ec.getMessage());
841 }
842 return returnVal;
843 } catch (IOException e) {
844 if (LOG.isDebugEnabled()) {
845 LOG.debug("Error while create symlink " + linkname + " to " + target
846 + "." + " Exception: " + StringUtils.stringifyException(e));
847 }
848 throw e;
849 }
850 return shExec.getExitCode();
851 }
852
853 /**
854 * Change the permissions on a filename.
855 * @param filename the name of the file to change
856 * @param perm the permission string
857 * @return the exit code from the command
858 * @throws IOException
859 * @throws InterruptedException
860 */
861 public static int chmod(String filename, String perm
862 ) throws IOException, InterruptedException {
863 return chmod(filename, perm, false);
864 }
865
866 /**
867 * Change the permissions on a file / directory, recursively, if
868 * needed.
869 * @param filename name of the file whose permissions are to change
870 * @param perm permission string
871 * @param recursive true, if permissions should be changed recursively
872 * @return the exit code from the command.
873 * @throws IOException
874 */
875 public static int chmod(String filename, String perm, boolean recursive)
876 throws IOException {
877
878 if (NativeIO.isAvailable()) {
879 final Configuration conf = new Configuration();
880 final FileSystem lfs = FileSystem.getLocal(conf);
881 final ChmodParser pp;
882 try {
883 pp = new ChmodParser(perm);
884 } catch (IllegalArgumentException e) {
885 throw new IOException("Invalid arg: " + perm, e);
886 }
887 final FileStatus fstat = lfs.getFileStatus(new Path(filename));
888
889 chmod(lfs, fstat, pp, recursive);
890 return 0;
891 } else {
892 String [] cmd = Shell.getSetPermissionCommand(perm, recursive);
893 String[] args = new String[cmd.length + 1];
894 System.arraycopy(cmd, 0, args, 0, cmd.length);
895 args[cmd.length] = new File(filename).getPath();
896 ShellCommandExecutor shExec = new ShellCommandExecutor(args);
897 try {
898 shExec.execute();
899 }catch(IOException e) {
900 if(LOG.isDebugEnabled()) {
901 LOG.debug("Error while changing permission : " + filename
902 +" Exception: " + StringUtils.stringifyException(e));
903 }
904 }
905 return shExec.getExitCode();
906 }
907 }
908
909 public static void chmod(FileSystem fs, FileStatus fstat, ChmodParser pp,
910 boolean recursive) throws IOException {
911
912 final short newMode = pp.applyNewPermission(fstat);
913 final short oldMode = fstat.getPermission().toShort();
914 if (newMode != oldMode) {
915 fs.setPermission(fstat.getPath(), new FsPermission(newMode));
916 }
917
918 if (recursive && fstat.isDir()) {
919 final FileStatus[] kids = fs.listStatus(fstat.getPath());
920 for (FileStatus kidStatus : kids) {
921 chmod(fs, kidStatus, pp, recursive);
922 }
923 }
924 }
925
926 /**
927 * Set the ownership on a file / directory. User name and group name
928 * cannot both be null.
929 * @param file the file to change
930 * @param username the new user owner name
931 * @param groupname the new group owner name
932 * @throws IOException
933 */
934 public static void setOwner(File file, String username,
935 String groupname) throws IOException {
936 if (username == null && groupname == null) {
937 throw new IOException("username == null && groupname == null");
938 }
939
940 if (NativeIO.isAvailable()) {
941 NativeIO.POSIX.chown(file.getCanonicalPath(), username, groupname);
942 } else {
943 String arg = (username == null ? "" : username)
944 + (groupname == null ? "" : ":" + groupname);
945 String [] cmd = Shell.getSetOwnerCommand(arg);
946 execCommand(file, cmd);
947 }
948 }
949
950 /**
951 * Platform independent implementation for {@link File#setReadable(boolean)}
952 * File#setReadable does not work as expected on Windows.
953 * @param f input file
954 * @param readable
955 * @return true on success, false otherwise
956 */
957 public static boolean setReadable(File f, boolean readable) {
958 if (Shell.WINDOWS) {
959 try {
960 String permission = readable ? "u+r" : "u-r";
961 FileUtil.chmod(f.getCanonicalPath(), permission, false);
962 return true;
963 } catch (IOException ex) {
964 return false;
965 }
966 } else {
967 return f.setReadable(readable);
968 }
969 }
970
971 /**
972 * Platform independent implementation for {@link File#setWritable(boolean)}
973 * File#setWritable does not work as expected on Windows.
974 * @param f input file
975 * @param writable
976 * @return true on success, false otherwise
977 */
978 public static boolean setWritable(File f, boolean writable) {
979 if (Shell.WINDOWS) {
980 try {
981 String permission = writable ? "u+w" : "u-w";
982 FileUtil.chmod(f.getCanonicalPath(), permission, false);
983 return true;
984 } catch (IOException ex) {
985 return false;
986 }
987 } else {
988 return f.setWritable(writable);
989 }
990 }
991
992 /**
993 * Platform independent implementation for {@link File#setExecutable(boolean)}
994 * File#setExecutable does not work as expected on Windows.
995 * Note: revoking execute permission on folders does not have the same
996 * behavior on Windows as on Unix platforms. Creating, deleting or renaming
997 * a file within that folder will still succeed on Windows.
998 * @param f input file
999 * @param executable
1000 * @return true on success, false otherwise
1001 */
1002 public static boolean setExecutable(File f, boolean executable) {
1003 if (Shell.WINDOWS) {
1004 try {
1005 String permission = executable ? "u+x" : "u-x";
1006 FileUtil.chmod(f.getCanonicalPath(), permission, false);
1007 return true;
1008 } catch (IOException ex) {
1009 return false;
1010 }
1011 } else {
1012 return f.setExecutable(executable);
1013 }
1014 }
1015
1016 /**
1017 * Platform independent implementation for {@link File#canRead()}
1018 * @param f input file
1019 * @return On Unix, same as {@link File#canRead()}
1020 * On Windows, true if process has read access on the path
1021 */
1022 public static boolean canRead(File f) {
1023 if (Shell.WINDOWS) {
1024 try {
1025 return NativeIO.Windows.access(f.getCanonicalPath(),
1026 NativeIO.Windows.AccessRight.ACCESS_READ);
1027 } catch (IOException e) {
1028 return false;
1029 }
1030 } else {
1031 return f.canRead();
1032 }
1033 }
1034
1035 /**
1036 * Platform independent implementation for {@link File#canWrite()}
1037 * @param f input file
1038 * @return On Unix, same as {@link File#canWrite()}
1039 * On Windows, true if process has write access on the path
1040 */
1041 public static boolean canWrite(File f) {
1042 if (Shell.WINDOWS) {
1043 try {
1044 return NativeIO.Windows.access(f.getCanonicalPath(),
1045 NativeIO.Windows.AccessRight.ACCESS_WRITE);
1046 } catch (IOException e) {
1047 return false;
1048 }
1049 } else {
1050 return f.canWrite();
1051 }
1052 }
1053
1054 /**
1055 * Platform independent implementation for {@link File#canExecute()}
1056 * @param f input file
1057 * @return On Unix, same as {@link File#canExecute()}
1058 * On Windows, true if process has execute access on the path
1059 */
1060 public static boolean canExecute(File f) {
1061 if (Shell.WINDOWS) {
1062 try {
1063 return NativeIO.Windows.access(f.getCanonicalPath(),
1064 NativeIO.Windows.AccessRight.ACCESS_EXECUTE);
1065 } catch (IOException e) {
1066 return false;
1067 }
1068 } else {
1069 return f.canExecute();
1070 }
1071 }
1072
1073 /**
1074 * Set permissions to the required value. Uses the java primitives instead
1075 * of forking if group == other.
1076 * @param f the file to change
1077 * @param permission the new permissions
1078 * @throws IOException
1079 */
1080 public static void setPermission(File f, FsPermission permission
1081 ) throws IOException {
1082 FsAction user = permission.getUserAction();
1083 FsAction group = permission.getGroupAction();
1084 FsAction other = permission.getOtherAction();
1085
1086 // use the native/fork if the group/other permissions are different
1087 // or if the native is available or on Windows
1088 if (group != other || NativeIO.isAvailable() || Shell.WINDOWS) {
1089 execSetPermission(f, permission);
1090 return;
1091 }
1092
1093 boolean rv = true;
1094
1095 // read perms
1096 rv = f.setReadable(group.implies(FsAction.READ), false);
1097 checkReturnValue(rv, f, permission);
1098 if (group.implies(FsAction.READ) != user.implies(FsAction.READ)) {
1099 rv = f.setReadable(user.implies(FsAction.READ), true);
1100 checkReturnValue(rv, f, permission);
1101 }
1102
1103 // write perms
1104 rv = f.setWritable(group.implies(FsAction.WRITE), false);
1105 checkReturnValue(rv, f, permission);
1106 if (group.implies(FsAction.WRITE) != user.implies(FsAction.WRITE)) {
1107 rv = f.setWritable(user.implies(FsAction.WRITE), true);
1108 checkReturnValue(rv, f, permission);
1109 }
1110
1111 // exec perms
1112 rv = f.setExecutable(group.implies(FsAction.EXECUTE), false);
1113 checkReturnValue(rv, f, permission);
1114 if (group.implies(FsAction.EXECUTE) != user.implies(FsAction.EXECUTE)) {
1115 rv = f.setExecutable(user.implies(FsAction.EXECUTE), true);
1116 checkReturnValue(rv, f, permission);
1117 }
1118 }
1119
1120 private static void checkReturnValue(boolean rv, File p,
1121 FsPermission permission
1122 ) throws IOException {
1123 if (!rv) {
1124 throw new IOException("Failed to set permissions of path: " + p +
1125 " to " +
1126 String.format("%04o", permission.toShort()));
1127 }
1128 }
1129
1130 private static void execSetPermission(File f,
1131 FsPermission permission
1132 ) throws IOException {
1133 if (NativeIO.isAvailable()) {
1134 NativeIO.POSIX.chmod(f.getCanonicalPath(), permission.toShort());
1135 } else {
1136 execCommand(f, Shell.getSetPermissionCommand(
1137 String.format("%04o", permission.toShort()), false));
1138 }
1139 }
1140
1141 static String execCommand(File f, String... cmd) throws IOException {
1142 String[] args = new String[cmd.length + 1];
1143 System.arraycopy(cmd, 0, args, 0, cmd.length);
1144 args[cmd.length] = f.getCanonicalPath();
1145 String output = Shell.execCommand(args);
1146 return output;
1147 }
1148
1149 /**
1150 * Create a tmp file for a base file.
1151 * @param basefile the base file of the tmp
1152 * @param prefix file name prefix of tmp
1153 * @param isDeleteOnExit if true, the tmp will be deleted when the VM exits
1154 * @return a newly created tmp file
1155 * @exception IOException If a tmp file cannot created
1156 * @see java.io.File#createTempFile(String, String, File)
1157 * @see java.io.File#deleteOnExit()
1158 */
1159 public static final File createLocalTempFile(final File basefile,
1160 final String prefix,
1161 final boolean isDeleteOnExit)
1162 throws IOException {
1163 File tmp = File.createTempFile(prefix + basefile.getName(),
1164 "", basefile.getParentFile());
1165 if (isDeleteOnExit) {
1166 tmp.deleteOnExit();
1167 }
1168 return tmp;
1169 }
1170
1171 /**
1172 * Move the src file to the name specified by target.
1173 * @param src the source file
1174 * @param target the target file
1175 * @exception IOException If this operation fails
1176 */
1177 public static void replaceFile(File src, File target) throws IOException {
1178 /* renameTo() has two limitations on Windows platform.
1179 * src.renameTo(target) fails if
1180 * 1) If target already exists OR
1181 * 2) If target is already open for reading/writing.
1182 */
1183 if (!src.renameTo(target)) {
1184 int retries = 5;
1185 while (target.exists() && !target.delete() && retries-- >= 0) {
1186 try {
1187 Thread.sleep(1000);
1188 } catch (InterruptedException e) {
1189 throw new IOException("replaceFile interrupted.");
1190 }
1191 }
1192 if (!src.renameTo(target)) {
1193 throw new IOException("Unable to rename " + src +
1194 " to " + target);
1195 }
1196 }
1197 }
1198
1199 /**
1200 * A wrapper for {@link File#listFiles()}. This java.io API returns null
1201 * when a dir is not a directory or for any I/O error. Instead of having
1202 * null check everywhere File#listFiles() is used, we will add utility API
1203 * to get around this problem. For the majority of cases where we prefer
1204 * an IOException to be thrown.
1205 * @param dir directory for which listing should be performed
1206 * @return list of files or empty list
1207 * @exception IOException for invalid directory or for a bad disk.
1208 */
1209 public static File[] listFiles(File dir) throws IOException {
1210 File[] files = dir.listFiles();
1211 if(files == null) {
1212 throw new IOException("Invalid directory or I/O error occurred for dir: "
1213 + dir.toString());
1214 }
1215 return files;
1216 }
1217
1218 /**
1219 * A wrapper for {@link File#list()}. This java.io API returns null
1220 * when a dir is not a directory or for any I/O error. Instead of having
1221 * null check everywhere File#list() is used, we will add utility API
1222 * to get around this problem. For the majority of cases where we prefer
1223 * an IOException to be thrown.
1224 * @param dir directory for which listing should be performed
1225 * @return list of file names or empty string list
1226 * @exception IOException for invalid directory or for a bad disk.
1227 */
1228 public static String[] list(File dir) throws IOException {
1229 String[] fileNames = dir.list();
1230 if(fileNames == null) {
1231 throw new IOException("Invalid directory or I/O error occurred for dir: "
1232 + dir.toString());
1233 }
1234 return fileNames;
1235 }
1236
1237 /**
1238 * Create a jar file at the given path, containing a manifest with a classpath
1239 * that references all specified entries.
1240 *
1241 * Some platforms may have an upper limit on command line length. For example,
1242 * the maximum command line length on Windows is 8191 characters, but the
1243 * length of the classpath may exceed this. To work around this limitation,
1244 * use this method to create a small intermediate jar with a manifest that
1245 * contains the full classpath. It returns the absolute path to the new jar,
1246 * which the caller may set as the classpath for a new process.
1247 *
1248 * Environment variable evaluation is not supported within a jar manifest, so
1249 * this method expands environment variables before inserting classpath entries
1250 * to the manifest. The method parses environment variables according to
1251 * platform-specific syntax (%VAR% on Windows, or $VAR otherwise). On Windows,
1252 * environment variables are case-insensitive. For example, %VAR% and %var%
1253 * evaluate to the same value.
1254 *
1255 * Specifying the classpath in a jar manifest does not support wildcards, so
1256 * this method expands wildcards internally. Any classpath entry that ends
1257 * with * is translated to all files at that path with extension .jar or .JAR.
1258 *
1259 * @param inputClassPath String input classpath to bundle into the jar manifest
1260 * @param pwd Path to working directory to save jar
1261 * @param callerEnv Map<String, String> caller's environment variables to use
1262 * for expansion
1263 * @return String absolute path to new jar
1264 * @throws IOException if there is an I/O error while writing the jar file
1265 */
1266 public static String createJarWithClassPath(String inputClassPath, Path pwd,
1267 Map<String, String> callerEnv) throws IOException {
1268 // Replace environment variables, case-insensitive on Windows
1269 @SuppressWarnings("unchecked")
1270 Map<String, String> env = Shell.WINDOWS ? new CaseInsensitiveMap(callerEnv) :
1271 callerEnv;
1272 String[] classPathEntries = inputClassPath.split(File.pathSeparator);
1273 for (int i = 0; i < classPathEntries.length; ++i) {
1274 classPathEntries[i] = StringUtils.replaceTokens(classPathEntries[i],
1275 StringUtils.ENV_VAR_PATTERN, env);
1276 }
1277 File workingDir = new File(pwd.toString());
1278 if (!workingDir.mkdirs()) {
1279 // If mkdirs returns false because the working directory already exists,
1280 // then this is acceptable. If it returns false due to some other I/O
1281 // error, then this method will fail later with an IOException while saving
1282 // the jar.
1283 LOG.debug("mkdirs false for " + workingDir + ", execution will continue");
1284 }
1285
1286 // Append all entries
1287 List<String> classPathEntryList = new ArrayList<String>(
1288 classPathEntries.length);
1289 for (String classPathEntry: classPathEntries) {
1290 if (classPathEntry.length() == 0) {
1291 continue;
1292 }
1293 if (classPathEntry.endsWith("*")) {
1294 // Append all jars that match the wildcard
1295 Path globPath = new Path(classPathEntry).suffix("{.jar,.JAR}");
1296 FileStatus[] wildcardJars = FileContext.getLocalFSFileContext().util()
1297 .globStatus(globPath);
1298 if (wildcardJars != null) {
1299 for (FileStatus wildcardJar: wildcardJars) {
1300 classPathEntryList.add(wildcardJar.getPath().toUri().toURL()
1301 .toExternalForm());
1302 }
1303 }
1304 } else {
1305 // Append just this entry
1306 File fileCpEntry = null;
1307 if(!new Path(classPathEntry).isAbsolute()) {
1308 fileCpEntry = new File(workingDir, classPathEntry);
1309 }
1310 else {
1311 fileCpEntry = new File(classPathEntry);
1312 }
1313 String classPathEntryUrl = fileCpEntry.toURI().toURL()
1314 .toExternalForm();
1315
1316 // File.toURI only appends trailing '/' if it can determine that it is a
1317 // directory that already exists. (See JavaDocs.) If this entry had a
1318 // trailing '/' specified by the caller, then guarantee that the
1319 // classpath entry in the manifest has a trailing '/', and thus refers to
1320 // a directory instead of a file. This can happen if the caller is
1321 // creating a classpath jar referencing a directory that hasn't been
1322 // created yet, but will definitely be created before running.
1323 if (classPathEntry.endsWith(Path.SEPARATOR) &&
1324 !classPathEntryUrl.endsWith(Path.SEPARATOR)) {
1325 classPathEntryUrl = classPathEntryUrl + Path.SEPARATOR;
1326 }
1327 classPathEntryList.add(classPathEntryUrl);
1328 }
1329 }
1330 String jarClassPath = StringUtils.join(" ", classPathEntryList);
1331
1332 // Create the manifest
1333 Manifest jarManifest = new Manifest();
1334 jarManifest.getMainAttributes().putValue(
1335 Attributes.Name.MANIFEST_VERSION.toString(), "1.0");
1336 jarManifest.getMainAttributes().putValue(
1337 Attributes.Name.CLASS_PATH.toString(), jarClassPath);
1338
1339 // Write the manifest to output JAR file
1340 File classPathJar = File.createTempFile("classpath-", ".jar", workingDir);
1341 FileOutputStream fos = null;
1342 BufferedOutputStream bos = null;
1343 JarOutputStream jos = null;
1344 try {
1345 fos = new FileOutputStream(classPathJar);
1346 bos = new BufferedOutputStream(fos);
1347 jos = new JarOutputStream(bos, jarManifest);
1348 } finally {
1349 IOUtils.cleanup(LOG, jos, bos, fos);
1350 }
1351
1352 return classPathJar.getCanonicalPath();
1353 }
1354 }