001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 019package org.apache.hadoop.fs; 020 021import java.io.*; 022import java.net.InetAddress; 023import java.net.URI; 024import java.net.UnknownHostException; 025import java.util.ArrayList; 026import java.util.Arrays; 027import java.util.Enumeration; 028import java.util.List; 029import java.util.Map; 030import java.util.jar.Attributes; 031import java.util.jar.JarOutputStream; 032import java.util.jar.Manifest; 033import java.util.zip.GZIPInputStream; 034import java.util.zip.ZipEntry; 035import java.util.zip.ZipFile; 036 037import org.apache.commons.collections.map.CaseInsensitiveMap; 038import org.apache.commons.compress.archivers.tar.TarArchiveEntry; 039import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; 040import org.apache.hadoop.classification.InterfaceAudience; 041import org.apache.hadoop.classification.InterfaceStability; 042import org.apache.hadoop.conf.Configuration; 043import org.apache.hadoop.fs.permission.FsAction; 044import org.apache.hadoop.fs.permission.FsPermission; 045import org.apache.hadoop.io.IOUtils; 046import org.apache.hadoop.io.nativeio.NativeIO; 047import org.apache.hadoop.util.StringUtils; 048import org.apache.hadoop.util.Shell; 049import org.apache.hadoop.util.Shell.ShellCommandExecutor; 050import org.apache.commons.logging.Log; 051import org.apache.commons.logging.LogFactory; 052 053/** 054 * A collection of file-processing util methods 055 */ 056@InterfaceAudience.Public 057@InterfaceStability.Evolving 058public class FileUtil { 059 060 private static final Log LOG = LogFactory.getLog(FileUtil.class); 061 062 /* The error code is defined in winutils to indicate insufficient 063 * privilege to create symbolic links. This value need to keep in 064 * sync with the constant of the same name in: 065 * "src\winutils\common.h" 066 * */ 067 public static final int SYMLINK_NO_PRIVILEGE = 2; 068 069 /** 070 * convert an array of FileStatus to an array of Path 071 * 072 * @param stats 073 * an array of FileStatus objects 074 * @return an array of paths corresponding to the input 075 */ 076 public static Path[] stat2Paths(FileStatus[] stats) { 077 if (stats == null) 078 return null; 079 Path[] ret = new Path[stats.length]; 080 for (int i = 0; i < stats.length; ++i) { 081 ret[i] = stats[i].getPath(); 082 } 083 return ret; 084 } 085 086 /** 087 * convert an array of FileStatus to an array of Path. 088 * If stats if null, return path 089 * @param stats 090 * an array of FileStatus objects 091 * @param path 092 * default path to return in stats is null 093 * @return an array of paths corresponding to the input 094 */ 095 public static Path[] stat2Paths(FileStatus[] stats, Path path) { 096 if (stats == null) 097 return new Path[]{path}; 098 else 099 return stat2Paths(stats); 100 } 101 102 /** 103 * Delete a directory and all its contents. If 104 * we return false, the directory may be partially-deleted. 105 * (1) If dir is symlink to a file, the symlink is deleted. The file pointed 106 * to by the symlink is not deleted. 107 * (2) If dir is symlink to a directory, symlink is deleted. The directory 108 * pointed to by symlink is not deleted. 109 * (3) If dir is a normal file, it is deleted. 110 * (4) If dir is a normal directory, then dir and all its contents recursively 111 * are deleted. 112 */ 113 public static boolean fullyDelete(final File dir) { 114 return fullyDelete(dir, false); 115 } 116 117 /** 118 * Delete a directory and all its contents. If 119 * we return false, the directory may be partially-deleted. 120 * (1) If dir is symlink to a file, the symlink is deleted. The file pointed 121 * to by the symlink is not deleted. 122 * (2) If dir is symlink to a directory, symlink is deleted. The directory 123 * pointed to by symlink is not deleted. 124 * (3) If dir is a normal file, it is deleted. 125 * (4) If dir is a normal directory, then dir and all its contents recursively 126 * are deleted. 127 * @param dir the file or directory to be deleted 128 * @param tryGrantPermissions true if permissions should be modified to delete a file. 129 * @return true on success false on failure. 130 */ 131 public static boolean fullyDelete(final File dir, boolean tryGrantPermissions) { 132 if (tryGrantPermissions) { 133 // try to chmod +rwx the parent folder of the 'dir': 134 File parent = dir.getParentFile(); 135 grantPermissions(parent); 136 } 137 if (deleteImpl(dir, false)) { 138 // dir is (a) normal file, (b) symlink to a file, (c) empty directory or 139 // (d) symlink to a directory 140 return true; 141 } 142 // handle nonempty directory deletion 143 if (!fullyDeleteContents(dir, tryGrantPermissions)) { 144 return false; 145 } 146 return deleteImpl(dir, true); 147 } 148 149 /** 150 * Returns the target of the given symlink. Returns the empty string if 151 * the given path does not refer to a symlink or there is an error 152 * accessing the symlink. 153 * @param f File representing the symbolic link. 154 * @return The target of the symbolic link, empty string on error or if not 155 * a symlink. 156 */ 157 public static String readLink(File f) { 158 /* NB: Use readSymbolicLink in java.nio.file.Path once available. Could 159 * use getCanonicalPath in File to get the target of the symlink but that 160 * does not indicate if the given path refers to a symlink. 161 */ 162 try { 163 return Shell.execCommand( 164 Shell.getReadlinkCommand(f.toString())).trim(); 165 } catch (IOException x) { 166 return ""; 167 } 168 } 169 170 /* 171 * Pure-Java implementation of "chmod +rwx f". 172 */ 173 private static void grantPermissions(final File f) { 174 FileUtil.setExecutable(f, true); 175 FileUtil.setReadable(f, true); 176 FileUtil.setWritable(f, true); 177 } 178 179 private static boolean deleteImpl(final File f, final boolean doLog) { 180 if (f == null) { 181 LOG.warn("null file argument."); 182 return false; 183 } 184 final boolean wasDeleted = f.delete(); 185 if (wasDeleted) { 186 return true; 187 } 188 final boolean ex = f.exists(); 189 if (doLog && ex) { 190 LOG.warn("Failed to delete file or dir [" 191 + f.getAbsolutePath() + "]: it still exists."); 192 } 193 return !ex; 194 } 195 196 /** 197 * Delete the contents of a directory, not the directory itself. If 198 * we return false, the directory may be partially-deleted. 199 * If dir is a symlink to a directory, all the contents of the actual 200 * directory pointed to by dir will be deleted. 201 */ 202 public static boolean fullyDeleteContents(final File dir) { 203 return fullyDeleteContents(dir, false); 204 } 205 206 /** 207 * Delete the contents of a directory, not the directory itself. If 208 * we return false, the directory may be partially-deleted. 209 * If dir is a symlink to a directory, all the contents of the actual 210 * directory pointed to by dir will be deleted. 211 * @param tryGrantPermissions if 'true', try grant +rwx permissions to this 212 * and all the underlying directories before trying to delete their contents. 213 */ 214 public static boolean fullyDeleteContents(final File dir, final boolean tryGrantPermissions) { 215 if (tryGrantPermissions) { 216 // to be able to list the dir and delete files from it 217 // we must grant the dir rwx permissions: 218 grantPermissions(dir); 219 } 220 boolean deletionSucceeded = true; 221 final File[] contents = dir.listFiles(); 222 if (contents != null) { 223 for (int i = 0; i < contents.length; i++) { 224 if (contents[i].isFile()) { 225 if (!deleteImpl(contents[i], true)) {// normal file or symlink to another file 226 deletionSucceeded = false; 227 continue; // continue deletion of other files/dirs under dir 228 } 229 } else { 230 // Either directory or symlink to another directory. 231 // Try deleting the directory as this might be a symlink 232 boolean b = false; 233 b = deleteImpl(contents[i], false); 234 if (b){ 235 //this was indeed a symlink or an empty directory 236 continue; 237 } 238 // if not an empty directory or symlink let 239 // fullydelete handle it. 240 if (!fullyDelete(contents[i], tryGrantPermissions)) { 241 deletionSucceeded = false; 242 // continue deletion of other files/dirs under dir 243 } 244 } 245 } 246 } 247 return deletionSucceeded; 248 } 249 250 /** 251 * Recursively delete a directory. 252 * 253 * @param fs {@link FileSystem} on which the path is present 254 * @param dir directory to recursively delete 255 * @throws IOException 256 * @deprecated Use {@link FileSystem#delete(Path, boolean)} 257 */ 258 @Deprecated 259 public static void fullyDelete(FileSystem fs, Path dir) 260 throws IOException { 261 fs.delete(dir, true); 262 } 263 264 // 265 // If the destination is a subdirectory of the source, then 266 // generate exception 267 // 268 private static void checkDependencies(FileSystem srcFS, 269 Path src, 270 FileSystem dstFS, 271 Path dst) 272 throws IOException { 273 if (srcFS == dstFS) { 274 String srcq = src.makeQualified(srcFS).toString() + Path.SEPARATOR; 275 String dstq = dst.makeQualified(dstFS).toString() + Path.SEPARATOR; 276 if (dstq.startsWith(srcq)) { 277 if (srcq.length() == dstq.length()) { 278 throw new IOException("Cannot copy " + src + " to itself."); 279 } else { 280 throw new IOException("Cannot copy " + src + " to its subdirectory " + 281 dst); 282 } 283 } 284 } 285 } 286 287 /** Copy files between FileSystems. */ 288 public static boolean copy(FileSystem srcFS, Path src, 289 FileSystem dstFS, Path dst, 290 boolean deleteSource, 291 Configuration conf) throws IOException { 292 return copy(srcFS, src, dstFS, dst, deleteSource, true, conf); 293 } 294 295 public static boolean copy(FileSystem srcFS, Path[] srcs, 296 FileSystem dstFS, Path dst, 297 boolean deleteSource, 298 boolean overwrite, Configuration conf) 299 throws IOException { 300 boolean gotException = false; 301 boolean returnVal = true; 302 StringBuilder exceptions = new StringBuilder(); 303 304 if (srcs.length == 1) 305 return copy(srcFS, srcs[0], dstFS, dst, deleteSource, overwrite, conf); 306 307 // Check if dest is directory 308 if (!dstFS.exists(dst)) { 309 throw new IOException("`" + dst +"': specified destination directory " + 310 "does not exist"); 311 } else { 312 FileStatus sdst = dstFS.getFileStatus(dst); 313 if (!sdst.isDirectory()) 314 throw new IOException("copying multiple files, but last argument `" + 315 dst + "' is not a directory"); 316 } 317 318 for (Path src : srcs) { 319 try { 320 if (!copy(srcFS, src, dstFS, dst, deleteSource, overwrite, conf)) 321 returnVal = false; 322 } catch (IOException e) { 323 gotException = true; 324 exceptions.append(e.getMessage()); 325 exceptions.append("\n"); 326 } 327 } 328 if (gotException) { 329 throw new IOException(exceptions.toString()); 330 } 331 return returnVal; 332 } 333 334 /** Copy files between FileSystems. */ 335 public static boolean copy(FileSystem srcFS, Path src, 336 FileSystem dstFS, Path dst, 337 boolean deleteSource, 338 boolean overwrite, 339 Configuration conf) throws IOException { 340 FileStatus fileStatus = srcFS.getFileStatus(src); 341 return copy(srcFS, fileStatus, dstFS, dst, deleteSource, overwrite, conf); 342 } 343 344 /** Copy files between FileSystems. */ 345 public static boolean copy(FileSystem srcFS, FileStatus srcStatus, 346 FileSystem dstFS, Path dst, 347 boolean deleteSource, 348 boolean overwrite, 349 Configuration conf) throws IOException { 350 Path src = srcStatus.getPath(); 351 dst = checkDest(src.getName(), dstFS, dst, overwrite); 352 if (srcStatus.isDirectory()) { 353 checkDependencies(srcFS, src, dstFS, dst); 354 if (!dstFS.mkdirs(dst)) { 355 return false; 356 } 357 FileStatus contents[] = srcFS.listStatus(src); 358 for (int i = 0; i < contents.length; i++) { 359 copy(srcFS, contents[i], dstFS, 360 new Path(dst, contents[i].getPath().getName()), 361 deleteSource, overwrite, conf); 362 } 363 } else { 364 InputStream in=null; 365 OutputStream out = null; 366 try { 367 in = srcFS.open(src); 368 out = dstFS.create(dst, overwrite); 369 IOUtils.copyBytes(in, out, conf, true); 370 } catch (IOException e) { 371 IOUtils.closeStream(out); 372 IOUtils.closeStream(in); 373 throw e; 374 } 375 } 376 if (deleteSource) { 377 return srcFS.delete(src, true); 378 } else { 379 return true; 380 } 381 382 } 383 384 /** Copy all files in a directory to one output file (merge). */ 385 public static boolean copyMerge(FileSystem srcFS, Path srcDir, 386 FileSystem dstFS, Path dstFile, 387 boolean deleteSource, 388 Configuration conf, String addString) throws IOException { 389 dstFile = checkDest(srcDir.getName(), dstFS, dstFile, false); 390 391 if (!srcFS.getFileStatus(srcDir).isDirectory()) 392 return false; 393 394 OutputStream out = dstFS.create(dstFile); 395 396 try { 397 FileStatus contents[] = srcFS.listStatus(srcDir); 398 Arrays.sort(contents); 399 for (int i = 0; i < contents.length; i++) { 400 if (contents[i].isFile()) { 401 InputStream in = srcFS.open(contents[i].getPath()); 402 try { 403 IOUtils.copyBytes(in, out, conf, false); 404 if (addString!=null) 405 out.write(addString.getBytes("UTF-8")); 406 407 } finally { 408 in.close(); 409 } 410 } 411 } 412 } finally { 413 out.close(); 414 } 415 416 417 if (deleteSource) { 418 return srcFS.delete(srcDir, true); 419 } else { 420 return true; 421 } 422 } 423 424 /** Copy local files to a FileSystem. */ 425 public static boolean copy(File src, 426 FileSystem dstFS, Path dst, 427 boolean deleteSource, 428 Configuration conf) throws IOException { 429 dst = checkDest(src.getName(), dstFS, dst, false); 430 431 if (src.isDirectory()) { 432 if (!dstFS.mkdirs(dst)) { 433 return false; 434 } 435 File contents[] = listFiles(src); 436 for (int i = 0; i < contents.length; i++) { 437 copy(contents[i], dstFS, new Path(dst, contents[i].getName()), 438 deleteSource, conf); 439 } 440 } else if (src.isFile()) { 441 InputStream in = null; 442 OutputStream out =null; 443 try { 444 in = new FileInputStream(src); 445 out = dstFS.create(dst); 446 IOUtils.copyBytes(in, out, conf); 447 } catch (IOException e) { 448 IOUtils.closeStream( out ); 449 IOUtils.closeStream( in ); 450 throw e; 451 } 452 } else { 453 throw new IOException(src.toString() + 454 ": No such file or directory"); 455 } 456 if (deleteSource) { 457 return FileUtil.fullyDelete(src); 458 } else { 459 return true; 460 } 461 } 462 463 /** Copy FileSystem files to local files. */ 464 public static boolean copy(FileSystem srcFS, Path src, 465 File dst, boolean deleteSource, 466 Configuration conf) throws IOException { 467 FileStatus filestatus = srcFS.getFileStatus(src); 468 return copy(srcFS, filestatus, dst, deleteSource, conf); 469 } 470 471 /** Copy FileSystem files to local files. */ 472 private static boolean copy(FileSystem srcFS, FileStatus srcStatus, 473 File dst, boolean deleteSource, 474 Configuration conf) throws IOException { 475 Path src = srcStatus.getPath(); 476 if (srcStatus.isDirectory()) { 477 if (!dst.mkdirs()) { 478 return false; 479 } 480 FileStatus contents[] = srcFS.listStatus(src); 481 for (int i = 0; i < contents.length; i++) { 482 copy(srcFS, contents[i], 483 new File(dst, contents[i].getPath().getName()), 484 deleteSource, conf); 485 } 486 } else { 487 InputStream in = srcFS.open(src); 488 IOUtils.copyBytes(in, new FileOutputStream(dst), conf); 489 } 490 if (deleteSource) { 491 return srcFS.delete(src, true); 492 } else { 493 return true; 494 } 495 } 496 497 private static Path checkDest(String srcName, FileSystem dstFS, Path dst, 498 boolean overwrite) throws IOException { 499 if (dstFS.exists(dst)) { 500 FileStatus sdst = dstFS.getFileStatus(dst); 501 if (sdst.isDirectory()) { 502 if (null == srcName) { 503 throw new IOException("Target " + dst + " is a directory"); 504 } 505 return checkDest(null, dstFS, new Path(dst, srcName), overwrite); 506 } else if (!overwrite) { 507 throw new IOException("Target " + dst + " already exists"); 508 } 509 } 510 return dst; 511 } 512 513 /** 514 * Convert a os-native filename to a path that works for the shell. 515 * @param filename The filename to convert 516 * @return The unix pathname 517 * @throws IOException on windows, there can be problems with the subprocess 518 */ 519 public static String makeShellPath(String filename) throws IOException { 520 return filename; 521 } 522 523 /** 524 * Convert a os-native filename to a path that works for the shell. 525 * @param file The filename to convert 526 * @return The unix pathname 527 * @throws IOException on windows, there can be problems with the subprocess 528 */ 529 public static String makeShellPath(File file) throws IOException { 530 return makeShellPath(file, false); 531 } 532 533 /** 534 * Convert a os-native filename to a path that works for the shell. 535 * @param file The filename to convert 536 * @param makeCanonicalPath 537 * Whether to make canonical path for the file passed 538 * @return The unix pathname 539 * @throws IOException on windows, there can be problems with the subprocess 540 */ 541 public static String makeShellPath(File file, boolean makeCanonicalPath) 542 throws IOException { 543 if (makeCanonicalPath) { 544 return makeShellPath(file.getCanonicalPath()); 545 } else { 546 return makeShellPath(file.toString()); 547 } 548 } 549 550 /** 551 * Takes an input dir and returns the du on that local directory. Very basic 552 * implementation. 553 * 554 * @param dir 555 * The input dir to get the disk space of this local dir 556 * @return The total disk space of the input local directory 557 */ 558 public static long getDU(File dir) { 559 long size = 0; 560 if (!dir.exists()) 561 return 0; 562 if (!dir.isDirectory()) { 563 return dir.length(); 564 } else { 565 File[] allFiles = dir.listFiles(); 566 if(allFiles != null) { 567 for (int i = 0; i < allFiles.length; i++) { 568 boolean isSymLink; 569 try { 570 isSymLink = org.apache.commons.io.FileUtils.isSymlink(allFiles[i]); 571 } catch(IOException ioe) { 572 isSymLink = true; 573 } 574 if(!isSymLink) { 575 size += getDU(allFiles[i]); 576 } 577 } 578 } 579 return size; 580 } 581 } 582 583 /** 584 * Given a File input it will unzip the file in a the unzip directory 585 * passed as the second parameter 586 * @param inFile The zip file as input 587 * @param unzipDir The unzip directory where to unzip the zip file. 588 * @throws IOException 589 */ 590 public static void unZip(File inFile, File unzipDir) throws IOException { 591 Enumeration<? extends ZipEntry> entries; 592 ZipFile zipFile = new ZipFile(inFile); 593 594 try { 595 entries = zipFile.entries(); 596 while (entries.hasMoreElements()) { 597 ZipEntry entry = entries.nextElement(); 598 if (!entry.isDirectory()) { 599 InputStream in = zipFile.getInputStream(entry); 600 try { 601 File file = new File(unzipDir, entry.getName()); 602 if (!file.getParentFile().mkdirs()) { 603 if (!file.getParentFile().isDirectory()) { 604 throw new IOException("Mkdirs failed to create " + 605 file.getParentFile().toString()); 606 } 607 } 608 OutputStream out = new FileOutputStream(file); 609 try { 610 byte[] buffer = new byte[8192]; 611 int i; 612 while ((i = in.read(buffer)) != -1) { 613 out.write(buffer, 0, i); 614 } 615 } finally { 616 out.close(); 617 } 618 } finally { 619 in.close(); 620 } 621 } 622 } 623 } finally { 624 zipFile.close(); 625 } 626 } 627 628 /** 629 * Given a Tar File as input it will untar the file in a the untar directory 630 * passed as the second parameter 631 * 632 * This utility will untar ".tar" files and ".tar.gz","tgz" files. 633 * 634 * @param inFile The tar file as input. 635 * @param untarDir The untar directory where to untar the tar file. 636 * @throws IOException 637 */ 638 public static void unTar(File inFile, File untarDir) throws IOException { 639 if (!untarDir.mkdirs()) { 640 if (!untarDir.isDirectory()) { 641 throw new IOException("Mkdirs failed to create " + untarDir); 642 } 643 } 644 645 boolean gzipped = inFile.toString().endsWith("gz"); 646 if(Shell.WINDOWS) { 647 // Tar is not native to Windows. Use simple Java based implementation for 648 // tests and simple tar archives 649 unTarUsingJava(inFile, untarDir, gzipped); 650 } 651 else { 652 // spawn tar utility to untar archive for full fledged unix behavior such 653 // as resolving symlinks in tar archives 654 unTarUsingTar(inFile, untarDir, gzipped); 655 } 656 } 657 658 private static void unTarUsingTar(File inFile, File untarDir, 659 boolean gzipped) throws IOException { 660 StringBuffer untarCommand = new StringBuffer(); 661 if (gzipped) { 662 untarCommand.append(" gzip -dc '"); 663 untarCommand.append(FileUtil.makeShellPath(inFile)); 664 untarCommand.append("' | ("); 665 } 666 untarCommand.append("cd '"); 667 untarCommand.append(FileUtil.makeShellPath(untarDir)); 668 untarCommand.append("' ; "); 669 untarCommand.append("tar -xf "); 670 671 if (gzipped) { 672 untarCommand.append(" -)"); 673 } else { 674 untarCommand.append(FileUtil.makeShellPath(inFile)); 675 } 676 String[] shellCmd = { "bash", "-c", untarCommand.toString() }; 677 ShellCommandExecutor shexec = new ShellCommandExecutor(shellCmd); 678 shexec.execute(); 679 int exitcode = shexec.getExitCode(); 680 if (exitcode != 0) { 681 throw new IOException("Error untarring file " + inFile + 682 ". Tar process exited with exit code " + exitcode); 683 } 684 } 685 686 private static void unTarUsingJava(File inFile, File untarDir, 687 boolean gzipped) throws IOException { 688 InputStream inputStream = null; 689 TarArchiveInputStream tis = null; 690 try { 691 if (gzipped) { 692 inputStream = new BufferedInputStream(new GZIPInputStream( 693 new FileInputStream(inFile))); 694 } else { 695 inputStream = new BufferedInputStream(new FileInputStream(inFile)); 696 } 697 698 tis = new TarArchiveInputStream(inputStream); 699 700 for (TarArchiveEntry entry = tis.getNextTarEntry(); entry != null;) { 701 unpackEntries(tis, entry, untarDir); 702 entry = tis.getNextTarEntry(); 703 } 704 } finally { 705 IOUtils.cleanup(LOG, tis, inputStream); 706 } 707 } 708 709 private static void unpackEntries(TarArchiveInputStream tis, 710 TarArchiveEntry entry, File outputDir) throws IOException { 711 if (entry.isDirectory()) { 712 File subDir = new File(outputDir, entry.getName()); 713 if (!subDir.mkdirs() && !subDir.isDirectory()) { 714 throw new IOException("Mkdirs failed to create tar internal dir " 715 + outputDir); 716 } 717 718 for (TarArchiveEntry e : entry.getDirectoryEntries()) { 719 unpackEntries(tis, e, subDir); 720 } 721 722 return; 723 } 724 725 File outputFile = new File(outputDir, entry.getName()); 726 if (!outputFile.getParentFile().exists()) { 727 if (!outputFile.getParentFile().mkdirs()) { 728 throw new IOException("Mkdirs failed to create tar internal dir " 729 + outputDir); 730 } 731 } 732 733 int count; 734 byte data[] = new byte[2048]; 735 try (BufferedOutputStream outputStream = new BufferedOutputStream( 736 new FileOutputStream(outputFile));) { 737 738 while ((count = tis.read(data)) != -1) { 739 outputStream.write(data, 0, count); 740 } 741 742 outputStream.flush(); 743 } 744 } 745 746 /** 747 * Class for creating hardlinks. 748 * Supports Unix, WindXP. 749 * @deprecated Use {@link org.apache.hadoop.fs.HardLink} 750 */ 751 @Deprecated 752 public static class HardLink extends org.apache.hadoop.fs.HardLink { 753 // This is a stub to assist with coordinated change between 754 // COMMON and HDFS projects. It will be removed after the 755 // corresponding change is committed to HDFS. 756 } 757 758 /** 759 * Create a soft link between a src and destination 760 * only on a local disk. HDFS does not support this. 761 * On Windows, when symlink creation fails due to security 762 * setting, we will log a warning. The return code in this 763 * case is 2. 764 * 765 * @param target the target for symlink 766 * @param linkname the symlink 767 * @return 0 on success 768 */ 769 public static int symLink(String target, String linkname) throws IOException{ 770 // Run the input paths through Java's File so that they are converted to the 771 // native OS form 772 File targetFile = new File( 773 Path.getPathWithoutSchemeAndAuthority(new Path(target)).toString()); 774 File linkFile = new File( 775 Path.getPathWithoutSchemeAndAuthority(new Path(linkname)).toString()); 776 777 // If not on Java7+, copy a file instead of creating a symlink since 778 // Java6 has close to no support for symlinks on Windows. Specifically 779 // File#length and File#renameTo do not work as expected. 780 // (see HADOOP-9061 for additional details) 781 // We still create symlinks for directories, since the scenario in this 782 // case is different. The directory content could change in which 783 // case the symlink loses its purpose (for example task attempt log folder 784 // is symlinked under userlogs and userlogs are generated afterwards). 785 if (Shell.WINDOWS && !Shell.isJava7OrAbove() && targetFile.isFile()) { 786 try { 787 LOG.warn("FileUtil#symlink: On Windows+Java6, copying file instead " + 788 "of creating a symlink. Copying " + target + " -> " + linkname); 789 790 if (!linkFile.getParentFile().exists()) { 791 LOG.warn("Parent directory " + linkFile.getParent() + 792 " does not exist."); 793 return 1; 794 } else { 795 org.apache.commons.io.FileUtils.copyFile(targetFile, linkFile); 796 } 797 } catch (IOException ex) { 798 LOG.warn("FileUtil#symlink failed to copy the file with error: " 799 + ex.getMessage()); 800 // Exit with non-zero exit code 801 return 1; 802 } 803 return 0; 804 } 805 806 String[] cmd = Shell.getSymlinkCommand( 807 targetFile.toString(), 808 linkFile.toString()); 809 810 ShellCommandExecutor shExec; 811 try { 812 if (Shell.WINDOWS && 813 linkFile.getParentFile() != null && 814 !new Path(target).isAbsolute()) { 815 // Relative links on Windows must be resolvable at the time of 816 // creation. To ensure this we run the shell command in the directory 817 // of the link. 818 // 819 shExec = new ShellCommandExecutor(cmd, linkFile.getParentFile()); 820 } else { 821 shExec = new ShellCommandExecutor(cmd); 822 } 823 shExec.execute(); 824 } catch (Shell.ExitCodeException ec) { 825 int returnVal = ec.getExitCode(); 826 if (Shell.WINDOWS && returnVal == SYMLINK_NO_PRIVILEGE) { 827 LOG.warn("Fail to create symbolic links on Windows. " 828 + "The default security settings in Windows disallow non-elevated " 829 + "administrators and all non-administrators from creating symbolic links. " 830 + "This behavior can be changed in the Local Security Policy management console"); 831 } else if (returnVal != 0) { 832 LOG.warn("Command '" + StringUtils.join(" ", cmd) + "' failed " 833 + returnVal + " with: " + ec.getMessage()); 834 } 835 return returnVal; 836 } catch (IOException e) { 837 if (LOG.isDebugEnabled()) { 838 LOG.debug("Error while create symlink " + linkname + " to " + target 839 + "." + " Exception: " + StringUtils.stringifyException(e)); 840 } 841 throw e; 842 } 843 return shExec.getExitCode(); 844 } 845 846 /** 847 * Change the permissions on a filename. 848 * @param filename the name of the file to change 849 * @param perm the permission string 850 * @return the exit code from the command 851 * @throws IOException 852 * @throws InterruptedException 853 */ 854 public static int chmod(String filename, String perm 855 ) throws IOException, InterruptedException { 856 return chmod(filename, perm, false); 857 } 858 859 /** 860 * Change the permissions on a file / directory, recursively, if 861 * needed. 862 * @param filename name of the file whose permissions are to change 863 * @param perm permission string 864 * @param recursive true, if permissions should be changed recursively 865 * @return the exit code from the command. 866 * @throws IOException 867 */ 868 public static int chmod(String filename, String perm, boolean recursive) 869 throws IOException { 870 String [] cmd = Shell.getSetPermissionCommand(perm, recursive); 871 String[] args = new String[cmd.length + 1]; 872 System.arraycopy(cmd, 0, args, 0, cmd.length); 873 args[cmd.length] = new File(filename).getPath(); 874 ShellCommandExecutor shExec = new ShellCommandExecutor(args); 875 try { 876 shExec.execute(); 877 }catch(IOException e) { 878 if(LOG.isDebugEnabled()) { 879 LOG.debug("Error while changing permission : " + filename 880 +" Exception: " + StringUtils.stringifyException(e)); 881 } 882 } 883 return shExec.getExitCode(); 884 } 885 886 /** 887 * Set the ownership on a file / directory. User name and group name 888 * cannot both be null. 889 * @param file the file to change 890 * @param username the new user owner name 891 * @param groupname the new group owner name 892 * @throws IOException 893 */ 894 public static void setOwner(File file, String username, 895 String groupname) throws IOException { 896 if (username == null && groupname == null) { 897 throw new IOException("username == null && groupname == null"); 898 } 899 String arg = (username == null ? "" : username) 900 + (groupname == null ? "" : ":" + groupname); 901 String [] cmd = Shell.getSetOwnerCommand(arg); 902 execCommand(file, cmd); 903 } 904 905 /** 906 * Platform independent implementation for {@link File#setReadable(boolean)} 907 * File#setReadable does not work as expected on Windows. 908 * @param f input file 909 * @param readable 910 * @return true on success, false otherwise 911 */ 912 public static boolean setReadable(File f, boolean readable) { 913 if (Shell.WINDOWS) { 914 try { 915 String permission = readable ? "u+r" : "u-r"; 916 FileUtil.chmod(f.getCanonicalPath(), permission, false); 917 return true; 918 } catch (IOException ex) { 919 return false; 920 } 921 } else { 922 return f.setReadable(readable); 923 } 924 } 925 926 /** 927 * Platform independent implementation for {@link File#setWritable(boolean)} 928 * File#setWritable does not work as expected on Windows. 929 * @param f input file 930 * @param writable 931 * @return true on success, false otherwise 932 */ 933 public static boolean setWritable(File f, boolean writable) { 934 if (Shell.WINDOWS) { 935 try { 936 String permission = writable ? "u+w" : "u-w"; 937 FileUtil.chmod(f.getCanonicalPath(), permission, false); 938 return true; 939 } catch (IOException ex) { 940 return false; 941 } 942 } else { 943 return f.setWritable(writable); 944 } 945 } 946 947 /** 948 * Platform independent implementation for {@link File#setExecutable(boolean)} 949 * File#setExecutable does not work as expected on Windows. 950 * Note: revoking execute permission on folders does not have the same 951 * behavior on Windows as on Unix platforms. Creating, deleting or renaming 952 * a file within that folder will still succeed on Windows. 953 * @param f input file 954 * @param executable 955 * @return true on success, false otherwise 956 */ 957 public static boolean setExecutable(File f, boolean executable) { 958 if (Shell.WINDOWS) { 959 try { 960 String permission = executable ? "u+x" : "u-x"; 961 FileUtil.chmod(f.getCanonicalPath(), permission, false); 962 return true; 963 } catch (IOException ex) { 964 return false; 965 } 966 } else { 967 return f.setExecutable(executable); 968 } 969 } 970 971 /** 972 * Platform independent implementation for {@link File#canRead()} 973 * @param f input file 974 * @return On Unix, same as {@link File#canRead()} 975 * On Windows, true if process has read access on the path 976 */ 977 public static boolean canRead(File f) { 978 if (Shell.WINDOWS) { 979 try { 980 return NativeIO.Windows.access(f.getCanonicalPath(), 981 NativeIO.Windows.AccessRight.ACCESS_READ); 982 } catch (IOException e) { 983 return false; 984 } 985 } else { 986 return f.canRead(); 987 } 988 } 989 990 /** 991 * Platform independent implementation for {@link File#canWrite()} 992 * @param f input file 993 * @return On Unix, same as {@link File#canWrite()} 994 * On Windows, true if process has write access on the path 995 */ 996 public static boolean canWrite(File f) { 997 if (Shell.WINDOWS) { 998 try { 999 return NativeIO.Windows.access(f.getCanonicalPath(), 1000 NativeIO.Windows.AccessRight.ACCESS_WRITE); 1001 } catch (IOException e) { 1002 return false; 1003 } 1004 } else { 1005 return f.canWrite(); 1006 } 1007 } 1008 1009 /** 1010 * Platform independent implementation for {@link File#canExecute()} 1011 * @param f input file 1012 * @return On Unix, same as {@link File#canExecute()} 1013 * On Windows, true if process has execute access on the path 1014 */ 1015 public static boolean canExecute(File f) { 1016 if (Shell.WINDOWS) { 1017 try { 1018 return NativeIO.Windows.access(f.getCanonicalPath(), 1019 NativeIO.Windows.AccessRight.ACCESS_EXECUTE); 1020 } catch (IOException e) { 1021 return false; 1022 } 1023 } else { 1024 return f.canExecute(); 1025 } 1026 } 1027 1028 /** 1029 * Set permissions to the required value. Uses the java primitives instead 1030 * of forking if group == other. 1031 * @param f the file to change 1032 * @param permission the new permissions 1033 * @throws IOException 1034 */ 1035 public static void setPermission(File f, FsPermission permission 1036 ) throws IOException { 1037 FsAction user = permission.getUserAction(); 1038 FsAction group = permission.getGroupAction(); 1039 FsAction other = permission.getOtherAction(); 1040 1041 // use the native/fork if the group/other permissions are different 1042 // or if the native is available or on Windows 1043 if (group != other || NativeIO.isAvailable() || Shell.WINDOWS) { 1044 execSetPermission(f, permission); 1045 return; 1046 } 1047 1048 boolean rv = true; 1049 1050 // read perms 1051 rv = f.setReadable(group.implies(FsAction.READ), false); 1052 checkReturnValue(rv, f, permission); 1053 if (group.implies(FsAction.READ) != user.implies(FsAction.READ)) { 1054 rv = f.setReadable(user.implies(FsAction.READ), true); 1055 checkReturnValue(rv, f, permission); 1056 } 1057 1058 // write perms 1059 rv = f.setWritable(group.implies(FsAction.WRITE), false); 1060 checkReturnValue(rv, f, permission); 1061 if (group.implies(FsAction.WRITE) != user.implies(FsAction.WRITE)) { 1062 rv = f.setWritable(user.implies(FsAction.WRITE), true); 1063 checkReturnValue(rv, f, permission); 1064 } 1065 1066 // exec perms 1067 rv = f.setExecutable(group.implies(FsAction.EXECUTE), false); 1068 checkReturnValue(rv, f, permission); 1069 if (group.implies(FsAction.EXECUTE) != user.implies(FsAction.EXECUTE)) { 1070 rv = f.setExecutable(user.implies(FsAction.EXECUTE), true); 1071 checkReturnValue(rv, f, permission); 1072 } 1073 } 1074 1075 private static void checkReturnValue(boolean rv, File p, 1076 FsPermission permission 1077 ) throws IOException { 1078 if (!rv) { 1079 throw new IOException("Failed to set permissions of path: " + p + 1080 " to " + 1081 String.format("%04o", permission.toShort())); 1082 } 1083 } 1084 1085 private static void execSetPermission(File f, 1086 FsPermission permission 1087 ) throws IOException { 1088 if (NativeIO.isAvailable()) { 1089 NativeIO.POSIX.chmod(f.getCanonicalPath(), permission.toShort()); 1090 } else { 1091 execCommand(f, Shell.getSetPermissionCommand( 1092 String.format("%04o", permission.toShort()), false)); 1093 } 1094 } 1095 1096 static String execCommand(File f, String... cmd) throws IOException { 1097 String[] args = new String[cmd.length + 1]; 1098 System.arraycopy(cmd, 0, args, 0, cmd.length); 1099 args[cmd.length] = f.getCanonicalPath(); 1100 String output = Shell.execCommand(args); 1101 return output; 1102 } 1103 1104 /** 1105 * Create a tmp file for a base file. 1106 * @param basefile the base file of the tmp 1107 * @param prefix file name prefix of tmp 1108 * @param isDeleteOnExit if true, the tmp will be deleted when the VM exits 1109 * @return a newly created tmp file 1110 * @exception IOException If a tmp file cannot created 1111 * @see java.io.File#createTempFile(String, String, File) 1112 * @see java.io.File#deleteOnExit() 1113 */ 1114 public static final File createLocalTempFile(final File basefile, 1115 final String prefix, 1116 final boolean isDeleteOnExit) 1117 throws IOException { 1118 File tmp = File.createTempFile(prefix + basefile.getName(), 1119 "", basefile.getParentFile()); 1120 if (isDeleteOnExit) { 1121 tmp.deleteOnExit(); 1122 } 1123 return tmp; 1124 } 1125 1126 /** 1127 * Move the src file to the name specified by target. 1128 * @param src the source file 1129 * @param target the target file 1130 * @exception IOException If this operation fails 1131 */ 1132 public static void replaceFile(File src, File target) throws IOException { 1133 /* renameTo() has two limitations on Windows platform. 1134 * src.renameTo(target) fails if 1135 * 1) If target already exists OR 1136 * 2) If target is already open for reading/writing. 1137 */ 1138 if (!src.renameTo(target)) { 1139 int retries = 5; 1140 while (target.exists() && !target.delete() && retries-- >= 0) { 1141 try { 1142 Thread.sleep(1000); 1143 } catch (InterruptedException e) { 1144 throw new IOException("replaceFile interrupted."); 1145 } 1146 } 1147 if (!src.renameTo(target)) { 1148 throw new IOException("Unable to rename " + src + 1149 " to " + target); 1150 } 1151 } 1152 } 1153 1154 /** 1155 * A wrapper for {@link File#listFiles()}. This java.io API returns null 1156 * when a dir is not a directory or for any I/O error. Instead of having 1157 * null check everywhere File#listFiles() is used, we will add utility API 1158 * to get around this problem. For the majority of cases where we prefer 1159 * an IOException to be thrown. 1160 * @param dir directory for which listing should be performed 1161 * @return list of files or empty list 1162 * @exception IOException for invalid directory or for a bad disk. 1163 */ 1164 public static File[] listFiles(File dir) throws IOException { 1165 File[] files = dir.listFiles(); 1166 if(files == null) { 1167 throw new IOException("Invalid directory or I/O error occurred for dir: " 1168 + dir.toString()); 1169 } 1170 return files; 1171 } 1172 1173 /** 1174 * A wrapper for {@link File#list()}. This java.io API returns null 1175 * when a dir is not a directory or for any I/O error. Instead of having 1176 * null check everywhere File#list() is used, we will add utility API 1177 * to get around this problem. For the majority of cases where we prefer 1178 * an IOException to be thrown. 1179 * @param dir directory for which listing should be performed 1180 * @return list of file names or empty string list 1181 * @exception IOException for invalid directory or for a bad disk. 1182 */ 1183 public static String[] list(File dir) throws IOException { 1184 String[] fileNames = dir.list(); 1185 if(fileNames == null) { 1186 throw new IOException("Invalid directory or I/O error occurred for dir: " 1187 + dir.toString()); 1188 } 1189 return fileNames; 1190 } 1191 1192 public static String[] createJarWithClassPath(String inputClassPath, Path pwd, 1193 Map<String, String> callerEnv) throws IOException { 1194 return createJarWithClassPath(inputClassPath, pwd, pwd, callerEnv); 1195 } 1196 1197 /** 1198 * Create a jar file at the given path, containing a manifest with a classpath 1199 * that references all specified entries. 1200 * 1201 * Some platforms may have an upper limit on command line length. For example, 1202 * the maximum command line length on Windows is 8191 characters, but the 1203 * length of the classpath may exceed this. To work around this limitation, 1204 * use this method to create a small intermediate jar with a manifest that 1205 * contains the full classpath. It returns the absolute path to the new jar, 1206 * which the caller may set as the classpath for a new process. 1207 * 1208 * Environment variable evaluation is not supported within a jar manifest, so 1209 * this method expands environment variables before inserting classpath entries 1210 * to the manifest. The method parses environment variables according to 1211 * platform-specific syntax (%VAR% on Windows, or $VAR otherwise). On Windows, 1212 * environment variables are case-insensitive. For example, %VAR% and %var% 1213 * evaluate to the same value. 1214 * 1215 * Specifying the classpath in a jar manifest does not support wildcards, so 1216 * this method expands wildcards internally. Any classpath entry that ends 1217 * with * is translated to all files at that path with extension .jar or .JAR. 1218 * 1219 * @param inputClassPath String input classpath to bundle into the jar manifest 1220 * @param pwd Path to working directory to save jar 1221 * @param targetDir path to where the jar execution will have its working dir 1222 * @param callerEnv Map<String, String> caller's environment variables to use 1223 * for expansion 1224 * @return String[] with absolute path to new jar in position 0 and 1225 * unexpanded wild card entry path in position 1 1226 * @throws IOException if there is an I/O error while writing the jar file 1227 */ 1228 public static String[] createJarWithClassPath(String inputClassPath, Path pwd, 1229 Path targetDir, 1230 Map<String, String> callerEnv) throws IOException { 1231 // Replace environment variables, case-insensitive on Windows 1232 @SuppressWarnings("unchecked") 1233 Map<String, String> env = Shell.WINDOWS ? new CaseInsensitiveMap(callerEnv) : 1234 callerEnv; 1235 String[] classPathEntries = inputClassPath.split(File.pathSeparator); 1236 for (int i = 0; i < classPathEntries.length; ++i) { 1237 classPathEntries[i] = StringUtils.replaceTokens(classPathEntries[i], 1238 StringUtils.ENV_VAR_PATTERN, env); 1239 } 1240 File workingDir = new File(pwd.toString()); 1241 if (!workingDir.mkdirs()) { 1242 // If mkdirs returns false because the working directory already exists, 1243 // then this is acceptable. If it returns false due to some other I/O 1244 // error, then this method will fail later with an IOException while saving 1245 // the jar. 1246 LOG.debug("mkdirs false for " + workingDir + ", execution will continue"); 1247 } 1248 1249 StringBuilder unexpandedWildcardClasspath = new StringBuilder(); 1250 // Append all entries 1251 List<String> classPathEntryList = new ArrayList<String>( 1252 classPathEntries.length); 1253 for (String classPathEntry: classPathEntries) { 1254 if (classPathEntry.length() == 0) { 1255 continue; 1256 } 1257 if (classPathEntry.endsWith("*")) { 1258 boolean foundWildCardJar = false; 1259 // Append all jars that match the wildcard 1260 Path globPath = new Path(classPathEntry).suffix("{.jar,.JAR}"); 1261 FileStatus[] wildcardJars = FileContext.getLocalFSFileContext().util() 1262 .globStatus(globPath); 1263 if (wildcardJars != null) { 1264 for (FileStatus wildcardJar: wildcardJars) { 1265 foundWildCardJar = true; 1266 classPathEntryList.add(wildcardJar.getPath().toUri().toURL() 1267 .toExternalForm()); 1268 } 1269 } 1270 if (!foundWildCardJar) { 1271 unexpandedWildcardClasspath.append(File.pathSeparator); 1272 unexpandedWildcardClasspath.append(classPathEntry); 1273 } 1274 } else { 1275 // Append just this entry 1276 File fileCpEntry = null; 1277 if(!new Path(classPathEntry).isAbsolute()) { 1278 fileCpEntry = new File(targetDir.toString(), classPathEntry); 1279 } 1280 else { 1281 fileCpEntry = new File(classPathEntry); 1282 } 1283 String classPathEntryUrl = fileCpEntry.toURI().toURL() 1284 .toExternalForm(); 1285 1286 // File.toURI only appends trailing '/' if it can determine that it is a 1287 // directory that already exists. (See JavaDocs.) If this entry had a 1288 // trailing '/' specified by the caller, then guarantee that the 1289 // classpath entry in the manifest has a trailing '/', and thus refers to 1290 // a directory instead of a file. This can happen if the caller is 1291 // creating a classpath jar referencing a directory that hasn't been 1292 // created yet, but will definitely be created before running. 1293 if (classPathEntry.endsWith(Path.SEPARATOR) && 1294 !classPathEntryUrl.endsWith(Path.SEPARATOR)) { 1295 classPathEntryUrl = classPathEntryUrl + Path.SEPARATOR; 1296 } 1297 classPathEntryList.add(classPathEntryUrl); 1298 } 1299 } 1300 String jarClassPath = StringUtils.join(" ", classPathEntryList); 1301 1302 // Create the manifest 1303 Manifest jarManifest = new Manifest(); 1304 jarManifest.getMainAttributes().putValue( 1305 Attributes.Name.MANIFEST_VERSION.toString(), "1.0"); 1306 jarManifest.getMainAttributes().putValue( 1307 Attributes.Name.CLASS_PATH.toString(), jarClassPath); 1308 1309 // Write the manifest to output JAR file 1310 File classPathJar = File.createTempFile("classpath-", ".jar", workingDir); 1311 FileOutputStream fos = null; 1312 BufferedOutputStream bos = null; 1313 JarOutputStream jos = null; 1314 try { 1315 fos = new FileOutputStream(classPathJar); 1316 bos = new BufferedOutputStream(fos); 1317 jos = new JarOutputStream(bos, jarManifest); 1318 } finally { 1319 IOUtils.cleanup(LOG, jos, bos, fos); 1320 } 1321 String[] jarCp = {classPathJar.getCanonicalPath(), 1322 unexpandedWildcardClasspath.toString()}; 1323 return jarCp; 1324 } 1325 1326 public static boolean compareFs(FileSystem srcFs, FileSystem destFs) { 1327 if (srcFs==null || destFs==null) { 1328 return false; 1329 } 1330 URI srcUri = srcFs.getUri(); 1331 URI dstUri = destFs.getUri(); 1332 if (srcUri.getScheme()==null) { 1333 return false; 1334 } 1335 if (!srcUri.getScheme().equals(dstUri.getScheme())) { 1336 return false; 1337 } 1338 String srcHost = srcUri.getHost(); 1339 String dstHost = dstUri.getHost(); 1340 if ((srcHost!=null) && (dstHost!=null)) { 1341 if (srcHost.equals(dstHost)) { 1342 return srcUri.getPort()==dstUri.getPort(); 1343 } 1344 try { 1345 srcHost = InetAddress.getByName(srcHost).getCanonicalHostName(); 1346 dstHost = InetAddress.getByName(dstHost).getCanonicalHostName(); 1347 } catch (UnknownHostException ue) { 1348 if (LOG.isDebugEnabled()) { 1349 LOG.debug("Could not compare file-systems. Unknown host: ", ue); 1350 } 1351 return false; 1352 } 1353 if (!srcHost.equals(dstHost)) { 1354 return false; 1355 } 1356 } else if (srcHost==null && dstHost!=null) { 1357 return false; 1358 } else if (srcHost!=null) { 1359 return false; 1360 } 1361 // check for ports 1362 return srcUri.getPort()==dstUri.getPort(); 1363 } 1364}