001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.fs; 019 020import java.io.FileNotFoundException; 021import java.io.IOException; 022import java.io.InputStream; 023import java.io.OutputStream; 024import java.net.URI; 025import java.security.PrivilegedExceptionAction; 026import java.util.ArrayList; 027import java.util.EnumSet; 028import java.util.HashSet; 029import java.util.IdentityHashMap; 030import java.util.List; 031import java.util.Map; 032import java.util.Set; 033import java.util.Stack; 034import java.util.TreeSet; 035import java.util.Map.Entry; 036 037import org.apache.commons.logging.Log; 038import org.apache.commons.logging.LogFactory; 039import org.apache.hadoop.HadoopIllegalArgumentException; 040import org.apache.hadoop.classification.InterfaceAudience; 041import org.apache.hadoop.classification.InterfaceStability; 042import org.apache.hadoop.conf.Configuration; 043import org.apache.hadoop.fs.FileSystem.Statistics; 044import org.apache.hadoop.fs.Options.CreateOpts; 045import org.apache.hadoop.fs.permission.AclEntry; 046import org.apache.hadoop.fs.permission.AclStatus; 047import org.apache.hadoop.fs.permission.FsAction; 048import org.apache.hadoop.fs.permission.FsCreateModes; 049import org.apache.hadoop.fs.permission.FsPermission; 050import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY; 051import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_DEFAULT; 052import org.apache.hadoop.io.IOUtils; 053import org.apache.hadoop.ipc.RpcClientException; 054import org.apache.hadoop.ipc.RpcServerException; 055import org.apache.hadoop.ipc.UnexpectedServerException; 056import org.apache.hadoop.fs.InvalidPathException; 057import org.apache.hadoop.security.AccessControlException; 058import org.apache.hadoop.security.UserGroupInformation; 059import org.apache.hadoop.security.token.Token; 060import org.apache.hadoop.util.ShutdownHookManager; 061import org.apache.htrace.core.Tracer; 062 063import com.google.common.base.Preconditions; 064 065/** 066 * The FileContext class provides an interface to the application writer for 067 * using the Hadoop file system. 068 * It provides a set of methods for the usual operation: create, open, 069 * list, etc 070 * 071 * <p> 072 * <b> *** Path Names *** </b> 073 * <p> 074 * 075 * The Hadoop file system supports a URI name space and URI names. 076 * It offers a forest of file systems that can be referenced using fully 077 * qualified URIs. 078 * Two common Hadoop file systems implementations are 079 * <ul> 080 * <li> the local file system: file:///path 081 * <li> the hdfs file system hdfs://nnAddress:nnPort/path 082 * </ul> 083 * 084 * While URI names are very flexible, it requires knowing the name or address 085 * of the server. For convenience one often wants to access the default system 086 * in one's environment without knowing its name/address. This has an 087 * additional benefit that it allows one to change one's default fs 088 * (e.g. admin moves application from cluster1 to cluster2). 089 * <p> 090 * 091 * To facilitate this, Hadoop supports a notion of a default file system. 092 * The user can set his default file system, although this is 093 * typically set up for you in your environment via your default config. 094 * A default file system implies a default scheme and authority; slash-relative 095 * names (such as /for/bar) are resolved relative to that default FS. 096 * Similarly a user can also have working-directory-relative names (i.e. names 097 * not starting with a slash). While the working directory is generally in the 098 * same default FS, the wd can be in a different FS. 099 * <p> 100 * Hence Hadoop path names can be one of: 101 * <ul> 102 * <li> fully qualified URI: scheme://authority/path 103 * <li> slash relative names: /path relative to the default file system 104 * <li> wd-relative names: path relative to the working dir 105 * </ul> 106 * Relative paths with scheme (scheme:foo/bar) are illegal. 107 * 108 * <p> 109 * <b>****The Role of the FileContext and configuration defaults****</b> 110 * <p> 111 * The FileContext provides file namespace context for resolving file names; 112 * it also contains the umask for permissions, In that sense it is like the 113 * per-process file-related state in Unix system. 114 * These two properties 115 * <ul> 116 * <li> default file system i.e your slash) 117 * <li> umask 118 * </ul> 119 * in general, are obtained from the default configuration file 120 * in your environment, (@see {@link Configuration}). 121 * 122 * No other configuration parameters are obtained from the default config as 123 * far as the file context layer is concerned. All file system instances 124 * (i.e. deployments of file systems) have default properties; we call these 125 * server side (SS) defaults. Operation like create allow one to select many 126 * properties: either pass them in as explicit parameters or use 127 * the SS properties. 128 * <p> 129 * The file system related SS defaults are 130 * <ul> 131 * <li> the home directory (default is "/user/userName") 132 * <li> the initial wd (only for local fs) 133 * <li> replication factor 134 * <li> block size 135 * <li> buffer size 136 * <li> encryptDataTransfer 137 * <li> checksum option. (checksumType and bytesPerChecksum) 138 * </ul> 139 * 140 * <p> 141 * <b> *** Usage Model for the FileContext class *** </b> 142 * <p> 143 * Example 1: use the default config read from the $HADOOP_CONFIG/core.xml. 144 * Unspecified values come from core-defaults.xml in the release jar. 145 * <ul> 146 * <li> myFContext = FileContext.getFileContext(); // uses the default config 147 * // which has your default FS 148 * <li> myFContext.create(path, ...); 149 * <li> myFContext.setWorkingDir(path) 150 * <li> myFContext.open (path, ...); 151 * </ul> 152 * Example 2: Get a FileContext with a specific URI as the default FS 153 * <ul> 154 * <li> myFContext = FileContext.getFileContext(URI) 155 * <li> myFContext.create(path, ...); 156 * ... 157 * </ul> 158 * Example 3: FileContext with local file system as the default 159 * <ul> 160 * <li> myFContext = FileContext.getLocalFSFileContext() 161 * <li> myFContext.create(path, ...); 162 * <li> ... 163 * </ul> 164 * Example 4: Use a specific config, ignoring $HADOOP_CONFIG 165 * Generally you should not need use a config unless you are doing 166 * <ul> 167 * <li> configX = someConfigSomeOnePassedToYou. 168 * <li> myFContext = getFileContext(configX); // configX is not changed, 169 * // is passed down 170 * <li> myFContext.create(path, ...); 171 * <li>... 172 * </ul> 173 * 174 */ 175 176@InterfaceAudience.Public 177@InterfaceStability.Evolving /*Evolving for a release,to be changed to Stable */ 178public class FileContext { 179 180 public static final Log LOG = LogFactory.getLog(FileContext.class); 181 /** 182 * Default permission for directory and symlink 183 * In previous versions, this default permission was also used to 184 * create files, so files created end up with ugo+x permission. 185 * See HADOOP-9155 for detail. 186 * Two new constants are added to solve this, please use 187 * {@link FileContext#DIR_DEFAULT_PERM} for directory, and use 188 * {@link FileContext#FILE_DEFAULT_PERM} for file. 189 * This constant is kept for compatibility. 190 */ 191 public static final FsPermission DEFAULT_PERM = FsPermission.getDefault(); 192 /** 193 * Default permission for directory 194 */ 195 public static final FsPermission DIR_DEFAULT_PERM = FsPermission.getDirDefault(); 196 /** 197 * Default permission for file 198 */ 199 public static final FsPermission FILE_DEFAULT_PERM = FsPermission.getFileDefault(); 200 201 /** 202 * Priority of the FileContext shutdown hook. 203 */ 204 public static final int SHUTDOWN_HOOK_PRIORITY = 20; 205 206 /** 207 * List of files that should be deleted on JVM shutdown. 208 */ 209 static final Map<FileContext, Set<Path>> DELETE_ON_EXIT = 210 new IdentityHashMap<FileContext, Set<Path>>(); 211 212 /** JVM shutdown hook thread. */ 213 static final FileContextFinalizer FINALIZER = 214 new FileContextFinalizer(); 215 216 private static final PathFilter DEFAULT_FILTER = new PathFilter() { 217 @Override 218 public boolean accept(final Path file) { 219 return true; 220 } 221 }; 222 223 /** 224 * The FileContext is defined by. 225 * 1) defaultFS (slash) 226 * 2) wd 227 * 3) umask 228 */ 229 private final AbstractFileSystem defaultFS; //default FS for this FileContext. 230 private Path workingDir; // Fully qualified 231 private FsPermission umask; 232 private final Configuration conf; 233 private final UserGroupInformation ugi; 234 final boolean resolveSymlinks; 235 private final Tracer tracer; 236 237 private FileContext(final AbstractFileSystem defFs, 238 final FsPermission theUmask, final Configuration aConf) { 239 defaultFS = defFs; 240 umask = FsPermission.getUMask(aConf); 241 conf = aConf; 242 tracer = FsTracer.get(aConf); 243 try { 244 ugi = UserGroupInformation.getCurrentUser(); 245 } catch (IOException e) { 246 LOG.error("Exception in getCurrentUser: ",e); 247 throw new RuntimeException("Failed to get the current user " + 248 "while creating a FileContext", e); 249 } 250 /* 251 * Init the wd. 252 * WorkingDir is implemented at the FileContext layer 253 * NOT at the AbstractFileSystem layer. 254 * If the DefaultFS, such as localFilesystem has a notion of 255 * builtin WD, we use that as the initial WD. 256 * Otherwise the WD is initialized to the home directory. 257 */ 258 workingDir = defaultFS.getInitialWorkingDirectory(); 259 if (workingDir == null) { 260 workingDir = defaultFS.getHomeDirectory(); 261 } 262 resolveSymlinks = conf.getBoolean( 263 CommonConfigurationKeys.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY, 264 CommonConfigurationKeys.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_DEFAULT); 265 util = new Util(); // for the inner class 266 } 267 268 /* 269 * Remove relative part - return "absolute": 270 * If input is relative path ("foo/bar") add wd: ie "/<workingDir>/foo/bar" 271 * A fully qualified uri ("hdfs://nn:p/foo/bar") or a slash-relative path 272 * ("/foo/bar") are returned unchanged. 273 * 274 * Applications that use FileContext should use #makeQualified() since 275 * they really want a fully qualified URI. 276 * Hence this method is not called makeAbsolute() and 277 * has been deliberately declared private. 278 */ 279 Path fixRelativePart(Path p) { 280 Preconditions.checkNotNull(p, "path cannot be null"); 281 if (p.isUriPathAbsolute()) { 282 return p; 283 } else { 284 return new Path(workingDir, p); 285 } 286 } 287 288 /** 289 * Delete all the paths that were marked as delete-on-exit. 290 */ 291 static void processDeleteOnExit() { 292 synchronized (DELETE_ON_EXIT) { 293 Set<Entry<FileContext, Set<Path>>> set = DELETE_ON_EXIT.entrySet(); 294 for (Entry<FileContext, Set<Path>> entry : set) { 295 FileContext fc = entry.getKey(); 296 Set<Path> paths = entry.getValue(); 297 for (Path path : paths) { 298 try { 299 fc.delete(path, true); 300 } catch (IOException e) { 301 LOG.warn("Ignoring failure to deleteOnExit for path " + path); 302 } 303 } 304 } 305 DELETE_ON_EXIT.clear(); 306 } 307 } 308 309 /** 310 * Get the file system of supplied path. 311 * 312 * @param absOrFqPath - absolute or fully qualified path 313 * @return the file system of the path 314 * 315 * @throws UnsupportedFileSystemException If the file system for 316 * <code>absOrFqPath</code> is not supported. 317 * @throws IOExcepton If the file system for <code>absOrFqPath</code> could 318 * not be instantiated. 319 */ 320 protected AbstractFileSystem getFSofPath(final Path absOrFqPath) 321 throws UnsupportedFileSystemException, IOException { 322 absOrFqPath.checkNotSchemeWithRelative(); 323 absOrFqPath.checkNotRelative(); 324 325 try { 326 // Is it the default FS for this FileContext? 327 defaultFS.checkPath(absOrFqPath); 328 return defaultFS; 329 } catch (Exception e) { // it is different FileSystem 330 return getAbstractFileSystem(ugi, absOrFqPath.toUri(), conf); 331 } 332 } 333 334 private static AbstractFileSystem getAbstractFileSystem( 335 UserGroupInformation user, final URI uri, final Configuration conf) 336 throws UnsupportedFileSystemException, IOException { 337 try { 338 return user.doAs(new PrivilegedExceptionAction<AbstractFileSystem>() { 339 @Override 340 public AbstractFileSystem run() throws UnsupportedFileSystemException { 341 return AbstractFileSystem.get(uri, conf); 342 } 343 }); 344 } catch (RuntimeException ex) { 345 // RTEs can wrap other exceptions; if there is an IOException inner, 346 // throw it direct. 347 Throwable cause = ex.getCause(); 348 if (cause instanceof IOException) { 349 throw (IOException) cause; 350 } else { 351 throw ex; 352 } 353 } catch (InterruptedException ex) { 354 LOG.error(ex); 355 throw new IOException("Failed to get the AbstractFileSystem for path: " 356 + uri, ex); 357 } 358 } 359 360 /** 361 * Protected Static Factory methods for getting a FileContexts 362 * that take a AbstractFileSystem as input. To be used for testing. 363 */ 364 365 /** 366 * Create a FileContext with specified FS as default using the specified 367 * config. 368 * 369 * @param defFS 370 * @param aConf 371 * @return new FileContext with specifed FS as default. 372 */ 373 public static FileContext getFileContext(final AbstractFileSystem defFS, 374 final Configuration aConf) { 375 return new FileContext(defFS, FsPermission.getUMask(aConf), aConf); 376 } 377 378 /** 379 * Create a FileContext for specified file system using the default config. 380 * 381 * @param defaultFS 382 * @return a FileContext with the specified AbstractFileSystem 383 * as the default FS. 384 */ 385 protected static FileContext getFileContext( 386 final AbstractFileSystem defaultFS) { 387 return getFileContext(defaultFS, new Configuration()); 388 } 389 390 /** 391 * Static Factory methods for getting a FileContext. 392 * Note new file contexts are created for each call. 393 * The only singleton is the local FS context using the default config. 394 * 395 * Methods that use the default config: the default config read from the 396 * $HADOOP_CONFIG/core.xml, 397 * Unspecified key-values for config are defaulted from core-defaults.xml 398 * in the release jar. 399 * 400 * The keys relevant to the FileContext layer are extracted at time of 401 * construction. Changes to the config after the call are ignore 402 * by the FileContext layer. 403 * The conf is passed to lower layers like AbstractFileSystem and HDFS which 404 * pick up their own config variables. 405 */ 406 407 /** 408 * Create a FileContext using the default config read from the 409 * $HADOOP_CONFIG/core.xml, Unspecified key-values for config are defaulted 410 * from core-defaults.xml in the release jar. 411 * 412 * @throws UnsupportedFileSystemException If the file system from the default 413 * configuration is not supported 414 */ 415 public static FileContext getFileContext() 416 throws UnsupportedFileSystemException { 417 return getFileContext(new Configuration()); 418 } 419 420 /** 421 * @return a FileContext for the local file system using the default config. 422 * @throws UnsupportedFileSystemException If the file system for 423 * {@link FsConstants#LOCAL_FS_URI} is not supported. 424 */ 425 public static FileContext getLocalFSFileContext() 426 throws UnsupportedFileSystemException { 427 return getFileContext(FsConstants.LOCAL_FS_URI); 428 } 429 430 /** 431 * Create a FileContext for specified URI using the default config. 432 * 433 * @param defaultFsUri 434 * @return a FileContext with the specified URI as the default FS. 435 * 436 * @throws UnsupportedFileSystemException If the file system for 437 * <code>defaultFsUri</code> is not supported 438 */ 439 public static FileContext getFileContext(final URI defaultFsUri) 440 throws UnsupportedFileSystemException { 441 return getFileContext(defaultFsUri, new Configuration()); 442 } 443 444 /** 445 * Create a FileContext for specified default URI using the specified config. 446 * 447 * @param defaultFsUri 448 * @param aConf 449 * @return new FileContext for specified uri 450 * @throws UnsupportedFileSystemException If the file system with specified is 451 * not supported 452 * @throws RuntimeException If the file system specified is supported but 453 * could not be instantiated, or if login fails. 454 */ 455 public static FileContext getFileContext(final URI defaultFsUri, 456 final Configuration aConf) throws UnsupportedFileSystemException { 457 UserGroupInformation currentUser = null; 458 AbstractFileSystem defaultAfs = null; 459 if (defaultFsUri.getScheme() == null) { 460 return getFileContext(aConf); 461 } 462 try { 463 currentUser = UserGroupInformation.getCurrentUser(); 464 defaultAfs = getAbstractFileSystem(currentUser, defaultFsUri, aConf); 465 } catch (UnsupportedFileSystemException ex) { 466 throw ex; 467 } catch (IOException ex) { 468 LOG.error(ex); 469 throw new RuntimeException(ex); 470 } 471 return getFileContext(defaultAfs, aConf); 472 } 473 474 /** 475 * Create a FileContext using the passed config. Generally it is better to use 476 * {@link #getFileContext(URI, Configuration)} instead of this one. 477 * 478 * 479 * @param aConf 480 * @return new FileContext 481 * @throws UnsupportedFileSystemException If file system in the config 482 * is not supported 483 */ 484 public static FileContext getFileContext(final Configuration aConf) 485 throws UnsupportedFileSystemException { 486 final URI defaultFsUri = URI.create(aConf.get(FS_DEFAULT_NAME_KEY, 487 FS_DEFAULT_NAME_DEFAULT)); 488 if ( defaultFsUri.getScheme() != null 489 && !defaultFsUri.getScheme().trim().isEmpty()) { 490 return getFileContext(defaultFsUri, aConf); 491 } 492 throw new UnsupportedFileSystemException(String.format( 493 "%s: URI configured via %s carries no scheme", 494 defaultFsUri, FS_DEFAULT_NAME_KEY)); 495 } 496 497 /** 498 * @param aConf - from which the FileContext is configured 499 * @return a FileContext for the local file system using the specified config. 500 * 501 * @throws UnsupportedFileSystemException If default file system in the config 502 * is not supported 503 * 504 */ 505 public static FileContext getLocalFSFileContext(final Configuration aConf) 506 throws UnsupportedFileSystemException { 507 return getFileContext(FsConstants.LOCAL_FS_URI, aConf); 508 } 509 510 /* This method is needed for tests. */ 511 @InterfaceAudience.Private 512 @InterfaceStability.Unstable /* return type will change to AFS once 513 HADOOP-6223 is completed */ 514 public AbstractFileSystem getDefaultFileSystem() { 515 return defaultFS; 516 } 517 518 /** 519 * Set the working directory for wd-relative names (such a "foo/bar"). Working 520 * directory feature is provided by simply prefixing relative names with the 521 * working dir. Note this is different from Unix where the wd is actually set 522 * to the inode. Hence setWorkingDir does not follow symlinks etc. This works 523 * better in a distributed environment that has multiple independent roots. 524 * {@link #getWorkingDirectory()} should return what setWorkingDir() set. 525 * 526 * @param newWDir new working directory 527 * @throws IOException 528 * <br> 529 * NewWdir can be one of: 530 * <ul> 531 * <li>relative path: "foo/bar";</li> 532 * <li>absolute without scheme: "/foo/bar"</li> 533 * <li>fully qualified with scheme: "xx://auth/foo/bar"</li> 534 * </ul> 535 * <br> 536 * Illegal WDs: 537 * <ul> 538 * <li>relative with scheme: "xx:foo/bar"</li> 539 * <li>non existent directory</li> 540 * </ul> 541 */ 542 public void setWorkingDirectory(final Path newWDir) throws IOException { 543 newWDir.checkNotSchemeWithRelative(); 544 /* wd is stored as a fully qualified path. We check if the given 545 * path is not relative first since resolve requires and returns 546 * an absolute path. 547 */ 548 final Path newWorkingDir = new Path(workingDir, newWDir); 549 FileStatus status = getFileStatus(newWorkingDir); 550 if (status.isFile()) { 551 throw new FileNotFoundException("Cannot setWD to a file"); 552 } 553 workingDir = newWorkingDir; 554 } 555 556 /** 557 * Gets the working directory for wd-relative names (such a "foo/bar"). 558 */ 559 public Path getWorkingDirectory() { 560 return workingDir; 561 } 562 563 /** 564 * Gets the ugi in the file-context 565 * @return UserGroupInformation 566 */ 567 public UserGroupInformation getUgi() { 568 return ugi; 569 } 570 571 /** 572 * Return the current user's home directory in this file system. 573 * The default implementation returns "/user/$USER/". 574 * @return the home directory 575 */ 576 public Path getHomeDirectory() { 577 return defaultFS.getHomeDirectory(); 578 } 579 580 /** 581 * 582 * @return the umask of this FileContext 583 */ 584 public FsPermission getUMask() { 585 return umask; 586 } 587 588 /** 589 * Set umask to the supplied parameter. 590 * @param newUmask the new umask 591 */ 592 public void setUMask(final FsPermission newUmask) { 593 umask = newUmask; 594 } 595 596 597 /** 598 * Resolve the path following any symlinks or mount points 599 * @param f to be resolved 600 * @return fully qualified resolved path 601 * 602 * @throws FileNotFoundException If <code>f</code> does not exist 603 * @throws AccessControlException if access denied 604 * @throws IOException If an IO Error occurred 605 * 606 * Exceptions applicable to file systems accessed over RPC: 607 * @throws RpcClientException If an exception occurred in the RPC client 608 * @throws RpcServerException If an exception occurred in the RPC server 609 * @throws UnexpectedServerException If server implementation throws 610 * undeclared exception to RPC server 611 * 612 * RuntimeExceptions: 613 * @throws InvalidPathException If path <code>f</code> is not valid 614 */ 615 public Path resolvePath(final Path f) throws FileNotFoundException, 616 UnresolvedLinkException, AccessControlException, IOException { 617 return resolve(f); 618 } 619 620 /** 621 * Make the path fully qualified if it is isn't. 622 * A Fully-qualified path has scheme and authority specified and an absolute 623 * path. 624 * Use the default file system and working dir in this FileContext to qualify. 625 * @param path 626 * @return qualified path 627 */ 628 public Path makeQualified(final Path path) { 629 return path.makeQualified(defaultFS.getUri(), getWorkingDirectory()); 630 } 631 632 /** 633 * Create or overwrite file on indicated path and returns an output stream for 634 * writing into the file. 635 * 636 * @param f the file name to open 637 * @param createFlag gives the semantics of create; see {@link CreateFlag} 638 * @param opts file creation options; see {@link Options.CreateOpts}. 639 * <ul> 640 * <li>Progress - to report progress on the operation - default null 641 * <li>Permission - umask is applied against permisssion: default is 642 * FsPermissions:getDefault() 643 * 644 * <li>CreateParent - create missing parent path; default is to not 645 * to create parents 646 * <li>The defaults for the following are SS defaults of the file 647 * server implementing the target path. Not all parameters make sense 648 * for all kinds of file system - eg. localFS ignores Blocksize, 649 * replication, checksum 650 * <ul> 651 * <li>BufferSize - buffersize used in FSDataOutputStream 652 * <li>Blocksize - block size for file blocks 653 * <li>ReplicationFactor - replication for blocks 654 * <li>ChecksumParam - Checksum parameters. server default is used 655 * if not specified. 656 * </ul> 657 * </ul> 658 * 659 * @return {@link FSDataOutputStream} for created file 660 * 661 * @throws AccessControlException If access is denied 662 * @throws FileAlreadyExistsException If file <code>f</code> already exists 663 * @throws FileNotFoundException If parent of <code>f</code> does not exist 664 * and <code>createParent</code> is false 665 * @throws ParentNotDirectoryException If parent of <code>f</code> is not a 666 * directory. 667 * @throws UnsupportedFileSystemException If file system for <code>f</code> is 668 * not supported 669 * @throws IOException If an I/O error occurred 670 * 671 * Exceptions applicable to file systems accessed over RPC: 672 * @throws RpcClientException If an exception occurred in the RPC client 673 * @throws RpcServerException If an exception occurred in the RPC server 674 * @throws UnexpectedServerException If server implementation throws 675 * undeclared exception to RPC server 676 * 677 * RuntimeExceptions: 678 * @throws InvalidPathException If path <code>f</code> is not valid 679 */ 680 public FSDataOutputStream create(final Path f, 681 final EnumSet<CreateFlag> createFlag, Options.CreateOpts... opts) 682 throws AccessControlException, FileAlreadyExistsException, 683 FileNotFoundException, ParentNotDirectoryException, 684 UnsupportedFileSystemException, IOException { 685 Path absF = fixRelativePart(f); 686 687 // If one of the options is a permission, extract it & apply umask 688 // If not, add a default Perms and apply umask; 689 // AbstractFileSystem#create 690 691 CreateOpts.Perms permOpt = CreateOpts.getOpt(CreateOpts.Perms.class, opts); 692 FsPermission permission = (permOpt != null) ? permOpt.getValue() : 693 FILE_DEFAULT_PERM; 694 permission = FsCreateModes.applyUMask(permission, umask); 695 696 final CreateOpts[] updatedOpts = 697 CreateOpts.setOpt(CreateOpts.perms(permission), opts); 698 return new FSLinkResolver<FSDataOutputStream>() { 699 @Override 700 public FSDataOutputStream next(final AbstractFileSystem fs, final Path p) 701 throws IOException { 702 return fs.create(p, createFlag, updatedOpts); 703 } 704 }.resolve(this, absF); 705 } 706 707 /** 708 * Make(create) a directory and all the non-existent parents. 709 * 710 * @param dir - the dir to make 711 * @param permission - permissions is set permission&~umask 712 * @param createParent - if true then missing parent dirs are created if false 713 * then parent must exist 714 * 715 * @throws AccessControlException If access is denied 716 * @throws FileAlreadyExistsException If directory <code>dir</code> already 717 * exists 718 * @throws FileNotFoundException If parent of <code>dir</code> does not exist 719 * and <code>createParent</code> is false 720 * @throws ParentNotDirectoryException If parent of <code>dir</code> is not a 721 * directory 722 * @throws UnsupportedFileSystemException If file system for <code>dir</code> 723 * is not supported 724 * @throws IOException If an I/O error occurred 725 * 726 * Exceptions applicable to file systems accessed over RPC: 727 * @throws RpcClientException If an exception occurred in the RPC client 728 * @throws UnexpectedServerException If server implementation throws 729 * undeclared exception to RPC server 730 * 731 * RuntimeExceptions: 732 * @throws InvalidPathException If path <code>dir</code> is not valid 733 */ 734 public void mkdir(final Path dir, final FsPermission permission, 735 final boolean createParent) throws AccessControlException, 736 FileAlreadyExistsException, FileNotFoundException, 737 ParentNotDirectoryException, UnsupportedFileSystemException, 738 IOException { 739 final Path absDir = fixRelativePart(dir); 740 final FsPermission absFerms = FsCreateModes.applyUMask( 741 permission == null ? 742 FsPermission.getDirDefault() : permission, umask); 743 new FSLinkResolver<Void>() { 744 @Override 745 public Void next(final AbstractFileSystem fs, final Path p) 746 throws IOException, UnresolvedLinkException { 747 fs.mkdir(p, absFerms, createParent); 748 return null; 749 } 750 }.resolve(this, absDir); 751 } 752 753 /** 754 * Delete a file. 755 * @param f the path to delete. 756 * @param recursive if path is a directory and set to 757 * true, the directory is deleted else throws an exception. In 758 * case of a file the recursive can be set to either true or false. 759 * 760 * @throws AccessControlException If access is denied 761 * @throws FileNotFoundException If <code>f</code> does not exist 762 * @throws UnsupportedFileSystemException If file system for <code>f</code> is 763 * not supported 764 * @throws IOException If an I/O error occurred 765 * 766 * Exceptions applicable to file systems accessed over RPC: 767 * @throws RpcClientException If an exception occurred in the RPC client 768 * @throws RpcServerException If an exception occurred in the RPC server 769 * @throws UnexpectedServerException If server implementation throws 770 * undeclared exception to RPC server 771 * 772 * RuntimeExceptions: 773 * @throws InvalidPathException If path <code>f</code> is invalid 774 */ 775 public boolean delete(final Path f, final boolean recursive) 776 throws AccessControlException, FileNotFoundException, 777 UnsupportedFileSystemException, IOException { 778 Path absF = fixRelativePart(f); 779 return new FSLinkResolver<Boolean>() { 780 @Override 781 public Boolean next(final AbstractFileSystem fs, final Path p) 782 throws IOException, UnresolvedLinkException { 783 return Boolean.valueOf(fs.delete(p, recursive)); 784 } 785 }.resolve(this, absF); 786 } 787 788 /** 789 * Opens an FSDataInputStream at the indicated Path using 790 * default buffersize. 791 * @param f the file name to open 792 * 793 * @throws AccessControlException If access is denied 794 * @throws FileNotFoundException If file <code>f</code> does not exist 795 * @throws UnsupportedFileSystemException If file system for <code>f</code> 796 * is not supported 797 * @throws IOException If an I/O error occurred 798 * 799 * Exceptions applicable to file systems accessed over RPC: 800 * @throws RpcClientException If an exception occurred in the RPC client 801 * @throws RpcServerException If an exception occurred in the RPC server 802 * @throws UnexpectedServerException If server implementation throws 803 * undeclared exception to RPC server 804 */ 805 public FSDataInputStream open(final Path f) throws AccessControlException, 806 FileNotFoundException, UnsupportedFileSystemException, IOException { 807 final Path absF = fixRelativePart(f); 808 return new FSLinkResolver<FSDataInputStream>() { 809 @Override 810 public FSDataInputStream next(final AbstractFileSystem fs, final Path p) 811 throws IOException, UnresolvedLinkException { 812 return fs.open(p); 813 } 814 }.resolve(this, absF); 815 } 816 817 /** 818 * Opens an FSDataInputStream at the indicated Path. 819 * 820 * @param f the file name to open 821 * @param bufferSize the size of the buffer to be used. 822 * 823 * @throws AccessControlException If access is denied 824 * @throws FileNotFoundException If file <code>f</code> does not exist 825 * @throws UnsupportedFileSystemException If file system for <code>f</code> is 826 * not supported 827 * @throws IOException If an I/O error occurred 828 * 829 * Exceptions applicable to file systems accessed over RPC: 830 * @throws RpcClientException If an exception occurred in the RPC client 831 * @throws RpcServerException If an exception occurred in the RPC server 832 * @throws UnexpectedServerException If server implementation throws 833 * undeclared exception to RPC server 834 */ 835 public FSDataInputStream open(final Path f, final int bufferSize) 836 throws AccessControlException, FileNotFoundException, 837 UnsupportedFileSystemException, IOException { 838 final Path absF = fixRelativePart(f); 839 return new FSLinkResolver<FSDataInputStream>() { 840 @Override 841 public FSDataInputStream next(final AbstractFileSystem fs, final Path p) 842 throws IOException, UnresolvedLinkException { 843 return fs.open(p, bufferSize); 844 } 845 }.resolve(this, absF); 846 } 847 848 /** 849 * Set replication for an existing file. 850 * 851 * @param f file name 852 * @param replication new replication 853 * 854 * @return true if successful 855 * 856 * @throws AccessControlException If access is denied 857 * @throws FileNotFoundException If file <code>f</code> does not exist 858 * @throws IOException If an I/O error occurred 859 * 860 * Exceptions applicable to file systems accessed over RPC: 861 * @throws RpcClientException If an exception occurred in the RPC client 862 * @throws RpcServerException If an exception occurred in the RPC server 863 * @throws UnexpectedServerException If server implementation throws 864 * undeclared exception to RPC server 865 */ 866 public boolean setReplication(final Path f, final short replication) 867 throws AccessControlException, FileNotFoundException, 868 IOException { 869 final Path absF = fixRelativePart(f); 870 return new FSLinkResolver<Boolean>() { 871 @Override 872 public Boolean next(final AbstractFileSystem fs, final Path p) 873 throws IOException, UnresolvedLinkException { 874 return Boolean.valueOf(fs.setReplication(p, replication)); 875 } 876 }.resolve(this, absF); 877 } 878 879 /** 880 * Renames Path src to Path dst 881 * <ul> 882 * <li 883 * <li>Fails if src is a file and dst is a directory. 884 * <li>Fails if src is a directory and dst is a file. 885 * <li>Fails if the parent of dst does not exist or is a file. 886 * </ul> 887 * <p> 888 * If OVERWRITE option is not passed as an argument, rename fails if the dst 889 * already exists. 890 * <p> 891 * If OVERWRITE option is passed as an argument, rename overwrites the dst if 892 * it is a file or an empty directory. Rename fails if dst is a non-empty 893 * directory. 894 * <p> 895 * Note that atomicity of rename is dependent on the file system 896 * implementation. Please refer to the file system documentation for details 897 * <p> 898 * 899 * @param src path to be renamed 900 * @param dst new path after rename 901 * 902 * @throws AccessControlException If access is denied 903 * @throws FileAlreadyExistsException If <code>dst</code> already exists and 904 * <code>options</options> has {@link Options.Rename#OVERWRITE} 905 * option false. 906 * @throws FileNotFoundException If <code>src</code> does not exist 907 * @throws ParentNotDirectoryException If parent of <code>dst</code> is not a 908 * directory 909 * @throws UnsupportedFileSystemException If file system for <code>src</code> 910 * and <code>dst</code> is not supported 911 * @throws IOException If an I/O error occurred 912 * 913 * Exceptions applicable to file systems accessed over RPC: 914 * @throws RpcClientException If an exception occurred in the RPC client 915 * @throws RpcServerException If an exception occurred in the RPC server 916 * @throws UnexpectedServerException If server implementation throws 917 * undeclared exception to RPC server 918 */ 919 public void rename(final Path src, final Path dst, 920 final Options.Rename... options) throws AccessControlException, 921 FileAlreadyExistsException, FileNotFoundException, 922 ParentNotDirectoryException, UnsupportedFileSystemException, 923 IOException { 924 final Path absSrc = fixRelativePart(src); 925 final Path absDst = fixRelativePart(dst); 926 AbstractFileSystem srcFS = getFSofPath(absSrc); 927 AbstractFileSystem dstFS = getFSofPath(absDst); 928 if(!srcFS.getUri().equals(dstFS.getUri())) { 929 throw new IOException("Renames across AbstractFileSystems not supported"); 930 } 931 try { 932 srcFS.rename(absSrc, absDst, options); 933 } catch (UnresolvedLinkException e) { 934 /* We do not know whether the source or the destination path 935 * was unresolved. Resolve the source path up until the final 936 * path component, then fully resolve the destination. 937 */ 938 final Path source = resolveIntermediate(absSrc); 939 new FSLinkResolver<Void>() { 940 @Override 941 public Void next(final AbstractFileSystem fs, final Path p) 942 throws IOException, UnresolvedLinkException { 943 fs.rename(source, p, options); 944 return null; 945 } 946 }.resolve(this, absDst); 947 } 948 } 949 950 /** 951 * Set permission of a path. 952 * @param f 953 * @param permission - the new absolute permission (umask is not applied) 954 * 955 * @throws AccessControlException If access is denied 956 * @throws FileNotFoundException If <code>f</code> does not exist 957 * @throws UnsupportedFileSystemException If file system for <code>f</code> 958 * is not supported 959 * @throws IOException If an I/O error occurred 960 * 961 * Exceptions applicable to file systems accessed over RPC: 962 * @throws RpcClientException If an exception occurred in the RPC client 963 * @throws RpcServerException If an exception occurred in the RPC server 964 * @throws UnexpectedServerException If server implementation throws 965 * undeclared exception to RPC server 966 */ 967 public void setPermission(final Path f, final FsPermission permission) 968 throws AccessControlException, FileNotFoundException, 969 UnsupportedFileSystemException, IOException { 970 final Path absF = fixRelativePart(f); 971 new FSLinkResolver<Void>() { 972 @Override 973 public Void next(final AbstractFileSystem fs, final Path p) 974 throws IOException, UnresolvedLinkException { 975 fs.setPermission(p, permission); 976 return null; 977 } 978 }.resolve(this, absF); 979 } 980 981 /** 982 * Set owner of a path (i.e. a file or a directory). The parameters username 983 * and groupname cannot both be null. 984 * 985 * @param f The path 986 * @param username If it is null, the original username remains unchanged. 987 * @param groupname If it is null, the original groupname remains unchanged. 988 * 989 * @throws AccessControlException If access is denied 990 * @throws FileNotFoundException If <code>f</code> does not exist 991 * @throws UnsupportedFileSystemException If file system for <code>f</code> is 992 * not supported 993 * @throws IOException If an I/O error occurred 994 * 995 * Exceptions applicable to file systems accessed over RPC: 996 * @throws RpcClientException If an exception occurred in the RPC client 997 * @throws RpcServerException If an exception occurred in the RPC server 998 * @throws UnexpectedServerException If server implementation throws 999 * undeclared exception to RPC server 1000 * 1001 * RuntimeExceptions: 1002 * @throws HadoopIllegalArgumentException If <code>username</code> or 1003 * <code>groupname</code> is invalid. 1004 */ 1005 public void setOwner(final Path f, final String username, 1006 final String groupname) throws AccessControlException, 1007 UnsupportedFileSystemException, FileNotFoundException, 1008 IOException { 1009 if ((username == null) && (groupname == null)) { 1010 throw new HadoopIllegalArgumentException( 1011 "username and groupname cannot both be null"); 1012 } 1013 final Path absF = fixRelativePart(f); 1014 new FSLinkResolver<Void>() { 1015 @Override 1016 public Void next(final AbstractFileSystem fs, final Path p) 1017 throws IOException, UnresolvedLinkException { 1018 fs.setOwner(p, username, groupname); 1019 return null; 1020 } 1021 }.resolve(this, absF); 1022 } 1023 1024 /** 1025 * Set access time of a file. 1026 * @param f The path 1027 * @param mtime Set the modification time of this file. 1028 * The number of milliseconds since epoch (Jan 1, 1970). 1029 * A value of -1 means that this call should not set modification time. 1030 * @param atime Set the access time of this file. 1031 * The number of milliseconds since Jan 1, 1970. 1032 * A value of -1 means that this call should not set access time. 1033 * 1034 * @throws AccessControlException If access is denied 1035 * @throws FileNotFoundException If <code>f</code> does not exist 1036 * @throws UnsupportedFileSystemException If file system for <code>f</code> is 1037 * not supported 1038 * @throws IOException If an I/O error occurred 1039 * 1040 * Exceptions applicable to file systems accessed over RPC: 1041 * @throws RpcClientException If an exception occurred in the RPC client 1042 * @throws RpcServerException If an exception occurred in the RPC server 1043 * @throws UnexpectedServerException If server implementation throws 1044 * undeclared exception to RPC server 1045 */ 1046 public void setTimes(final Path f, final long mtime, final long atime) 1047 throws AccessControlException, FileNotFoundException, 1048 UnsupportedFileSystemException, IOException { 1049 final Path absF = fixRelativePart(f); 1050 new FSLinkResolver<Void>() { 1051 @Override 1052 public Void next(final AbstractFileSystem fs, final Path p) 1053 throws IOException, UnresolvedLinkException { 1054 fs.setTimes(p, mtime, atime); 1055 return null; 1056 } 1057 }.resolve(this, absF); 1058 } 1059 1060 /** 1061 * Get the checksum of a file. 1062 * 1063 * @param f file path 1064 * 1065 * @return The file checksum. The default return value is null, 1066 * which indicates that no checksum algorithm is implemented 1067 * in the corresponding FileSystem. 1068 * 1069 * @throws AccessControlException If access is denied 1070 * @throws FileNotFoundException If <code>f</code> does not exist 1071 * @throws IOException If an I/O error occurred 1072 * 1073 * Exceptions applicable to file systems accessed over RPC: 1074 * @throws RpcClientException If an exception occurred in the RPC client 1075 * @throws RpcServerException If an exception occurred in the RPC server 1076 * @throws UnexpectedServerException If server implementation throws 1077 * undeclared exception to RPC server 1078 */ 1079 public FileChecksum getFileChecksum(final Path f) 1080 throws AccessControlException, FileNotFoundException, 1081 IOException { 1082 final Path absF = fixRelativePart(f); 1083 return new FSLinkResolver<FileChecksum>() { 1084 @Override 1085 public FileChecksum next(final AbstractFileSystem fs, final Path p) 1086 throws IOException, UnresolvedLinkException { 1087 return fs.getFileChecksum(p); 1088 } 1089 }.resolve(this, absF); 1090 } 1091 1092 /** 1093 * Set the verify checksum flag for the file system denoted by the path. 1094 * This is only applicable if the 1095 * corresponding FileSystem supports checksum. By default doesn't do anything. 1096 * @param verifyChecksum 1097 * @param f set the verifyChecksum for the Filesystem containing this path 1098 * 1099 * @throws AccessControlException If access is denied 1100 * @throws FileNotFoundException If <code>f</code> does not exist 1101 * @throws UnsupportedFileSystemException If file system for <code>f</code> is 1102 * not supported 1103 * @throws IOException If an I/O error occurred 1104 * 1105 * Exceptions applicable to file systems accessed over RPC: 1106 * @throws RpcClientException If an exception occurred in the RPC client 1107 * @throws RpcServerException If an exception occurred in the RPC server 1108 * @throws UnexpectedServerException If server implementation throws 1109 * undeclared exception to RPC server 1110 */ 1111 public void setVerifyChecksum(final boolean verifyChecksum, final Path f) 1112 throws AccessControlException, FileNotFoundException, 1113 UnsupportedFileSystemException, IOException { 1114 final Path absF = resolve(fixRelativePart(f)); 1115 getFSofPath(absF).setVerifyChecksum(verifyChecksum); 1116 } 1117 1118 /** 1119 * Return a file status object that represents the path. 1120 * @param f The path we want information from 1121 * 1122 * @return a FileStatus object 1123 * 1124 * @throws AccessControlException If access is denied 1125 * @throws FileNotFoundException If <code>f</code> does not exist 1126 * @throws UnsupportedFileSystemException If file system for <code>f</code> is 1127 * not supported 1128 * @throws IOException If an I/O error occurred 1129 * 1130 * Exceptions applicable to file systems accessed over RPC: 1131 * @throws RpcClientException If an exception occurred in the RPC client 1132 * @throws RpcServerException If an exception occurred in the RPC server 1133 * @throws UnexpectedServerException If server implementation throws 1134 * undeclared exception to RPC server 1135 */ 1136 public FileStatus getFileStatus(final Path f) throws AccessControlException, 1137 FileNotFoundException, UnsupportedFileSystemException, IOException { 1138 final Path absF = fixRelativePart(f); 1139 return new FSLinkResolver<FileStatus>() { 1140 @Override 1141 public FileStatus next(final AbstractFileSystem fs, final Path p) 1142 throws IOException, UnresolvedLinkException { 1143 return fs.getFileStatus(p); 1144 } 1145 }.resolve(this, absF); 1146 } 1147 1148 /** 1149 * Checks if the user can access a path. The mode specifies which access 1150 * checks to perform. If the requested permissions are granted, then the 1151 * method returns normally. If access is denied, then the method throws an 1152 * {@link AccessControlException}. 1153 * <p/> 1154 * The default implementation of this method calls {@link #getFileStatus(Path)} 1155 * and checks the returned permissions against the requested permissions. 1156 * Note that the getFileStatus call will be subject to authorization checks. 1157 * Typically, this requires search (execute) permissions on each directory in 1158 * the path's prefix, but this is implementation-defined. Any file system 1159 * that provides a richer authorization model (such as ACLs) may override the 1160 * default implementation so that it checks against that model instead. 1161 * <p> 1162 * In general, applications should avoid using this method, due to the risk of 1163 * time-of-check/time-of-use race conditions. The permissions on a file may 1164 * change immediately after the access call returns. Most applications should 1165 * prefer running specific file system actions as the desired user represented 1166 * by a {@link UserGroupInformation}. 1167 * 1168 * @param path Path to check 1169 * @param mode type of access to check 1170 * @throws AccessControlException if access is denied 1171 * @throws FileNotFoundException if the path does not exist 1172 * @throws UnsupportedFileSystemException if file system for <code>path</code> 1173 * is not supported 1174 * @throws IOException see specific implementation 1175 * 1176 * Exceptions applicable to file systems accessed over RPC: 1177 * @throws RpcClientException If an exception occurred in the RPC client 1178 * @throws RpcServerException If an exception occurred in the RPC server 1179 * @throws UnexpectedServerException If server implementation throws 1180 * undeclared exception to RPC server 1181 */ 1182 @InterfaceAudience.LimitedPrivate({"HDFS", "Hive"}) 1183 public void access(final Path path, final FsAction mode) 1184 throws AccessControlException, FileNotFoundException, 1185 UnsupportedFileSystemException, IOException { 1186 final Path absPath = fixRelativePart(path); 1187 new FSLinkResolver<Void>() { 1188 @Override 1189 public Void next(AbstractFileSystem fs, Path p) throws IOException, 1190 UnresolvedLinkException { 1191 fs.access(p, mode); 1192 return null; 1193 } 1194 }.resolve(this, absPath); 1195 } 1196 1197 /** 1198 * Return a file status object that represents the path. If the path 1199 * refers to a symlink then the FileStatus of the symlink is returned. 1200 * The behavior is equivalent to #getFileStatus() if the underlying 1201 * file system does not support symbolic links. 1202 * @param f The path we want information from. 1203 * @return A FileStatus object 1204 * 1205 * @throws AccessControlException If access is denied 1206 * @throws FileNotFoundException If <code>f</code> does not exist 1207 * @throws UnsupportedFileSystemException If file system for <code>f</code> is 1208 * not supported 1209 * @throws IOException If an I/O error occurred 1210 */ 1211 public FileStatus getFileLinkStatus(final Path f) 1212 throws AccessControlException, FileNotFoundException, 1213 UnsupportedFileSystemException, IOException { 1214 final Path absF = fixRelativePart(f); 1215 return new FSLinkResolver<FileStatus>() { 1216 @Override 1217 public FileStatus next(final AbstractFileSystem fs, final Path p) 1218 throws IOException, UnresolvedLinkException { 1219 FileStatus fi = fs.getFileLinkStatus(p); 1220 if (fi.isSymlink()) { 1221 fi.setSymlink(FSLinkResolver.qualifySymlinkTarget(fs.getUri(), p, 1222 fi.getSymlink())); 1223 } 1224 return fi; 1225 } 1226 }.resolve(this, absF); 1227 } 1228 1229 /** 1230 * Returns the target of the given symbolic link as it was specified 1231 * when the link was created. Links in the path leading up to the 1232 * final path component are resolved transparently. 1233 * 1234 * @param f the path to return the target of 1235 * @return The un-interpreted target of the symbolic link. 1236 * 1237 * @throws AccessControlException If access is denied 1238 * @throws FileNotFoundException If path <code>f</code> does not exist 1239 * @throws UnsupportedFileSystemException If file system for <code>f</code> is 1240 * not supported 1241 * @throws IOException If the given path does not refer to a symlink 1242 * or an I/O error occurred 1243 */ 1244 public Path getLinkTarget(final Path f) throws AccessControlException, 1245 FileNotFoundException, UnsupportedFileSystemException, IOException { 1246 final Path absF = fixRelativePart(f); 1247 return new FSLinkResolver<Path>() { 1248 @Override 1249 public Path next(final AbstractFileSystem fs, final Path p) 1250 throws IOException, UnresolvedLinkException { 1251 FileStatus fi = fs.getFileLinkStatus(p); 1252 return fi.getSymlink(); 1253 } 1254 }.resolve(this, absF); 1255 } 1256 1257 /** 1258 * Return blockLocation of the given file for the given offset and len. 1259 * For a nonexistent file or regions, null will be returned. 1260 * 1261 * This call is most helpful with DFS, where it returns 1262 * hostnames of machines that contain the given file. 1263 * 1264 * @param f - get blocklocations of this file 1265 * @param start position (byte offset) 1266 * @param len (in bytes) 1267 * 1268 * @return block locations for given file at specified offset of len 1269 * 1270 * @throws AccessControlException If access is denied 1271 * @throws FileNotFoundException If <code>f</code> does not exist 1272 * @throws UnsupportedFileSystemException If file system for <code>f</code> is 1273 * not supported 1274 * @throws IOException If an I/O error occurred 1275 * 1276 * Exceptions applicable to file systems accessed over RPC: 1277 * @throws RpcClientException If an exception occurred in the RPC client 1278 * @throws RpcServerException If an exception occurred in the RPC server 1279 * @throws UnexpectedServerException If server implementation throws 1280 * undeclared exception to RPC server 1281 * 1282 * RuntimeExceptions: 1283 * @throws InvalidPathException If path <code>f</code> is invalid 1284 */ 1285 @InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"}) 1286 @InterfaceStability.Evolving 1287 public BlockLocation[] getFileBlockLocations(final Path f, final long start, 1288 final long len) throws AccessControlException, FileNotFoundException, 1289 UnsupportedFileSystemException, IOException { 1290 final Path absF = fixRelativePart(f); 1291 return new FSLinkResolver<BlockLocation[]>() { 1292 @Override 1293 public BlockLocation[] next(final AbstractFileSystem fs, final Path p) 1294 throws IOException, UnresolvedLinkException { 1295 return fs.getFileBlockLocations(p, start, len); 1296 } 1297 }.resolve(this, absF); 1298 } 1299 1300 /** 1301 * Returns a status object describing the use and capacity of the 1302 * file system denoted by the Parh argument p. 1303 * If the file system has multiple partitions, the 1304 * use and capacity of the partition pointed to by the specified 1305 * path is reflected. 1306 * 1307 * @param f Path for which status should be obtained. null means the 1308 * root partition of the default file system. 1309 * 1310 * @return a FsStatus object 1311 * 1312 * @throws AccessControlException If access is denied 1313 * @throws FileNotFoundException If <code>f</code> does not exist 1314 * @throws UnsupportedFileSystemException If file system for <code>f</code> is 1315 * not supported 1316 * @throws IOException If an I/O error occurred 1317 * 1318 * Exceptions applicable to file systems accessed over RPC: 1319 * @throws RpcClientException If an exception occurred in the RPC client 1320 * @throws RpcServerException If an exception occurred in the RPC server 1321 * @throws UnexpectedServerException If server implementation throws 1322 * undeclared exception to RPC server 1323 */ 1324 public FsStatus getFsStatus(final Path f) throws AccessControlException, 1325 FileNotFoundException, UnsupportedFileSystemException, IOException { 1326 if (f == null) { 1327 return defaultFS.getFsStatus(); 1328 } 1329 final Path absF = fixRelativePart(f); 1330 return new FSLinkResolver<FsStatus>() { 1331 @Override 1332 public FsStatus next(final AbstractFileSystem fs, final Path p) 1333 throws IOException, UnresolvedLinkException { 1334 return fs.getFsStatus(p); 1335 } 1336 }.resolve(this, absF); 1337 } 1338 1339 /** 1340 * Creates a symbolic link to an existing file. An exception is thrown if 1341 * the symlink exits, the user does not have permission to create symlink, 1342 * or the underlying file system does not support symlinks. 1343 * 1344 * Symlink permissions are ignored, access to a symlink is determined by 1345 * the permissions of the symlink target. 1346 * 1347 * Symlinks in paths leading up to the final path component are resolved 1348 * transparently. If the final path component refers to a symlink some 1349 * functions operate on the symlink itself, these are: 1350 * - delete(f) and deleteOnExit(f) - Deletes the symlink. 1351 * - rename(src, dst) - If src refers to a symlink, the symlink is 1352 * renamed. If dst refers to a symlink, the symlink is over-written. 1353 * - getLinkTarget(f) - Returns the target of the symlink. 1354 * - getFileLinkStatus(f) - Returns a FileStatus object describing 1355 * the symlink. 1356 * Some functions, create() and mkdir(), expect the final path component 1357 * does not exist. If they are given a path that refers to a symlink that 1358 * does exist they behave as if the path referred to an existing file or 1359 * directory. All other functions fully resolve, ie follow, the symlink. 1360 * These are: open, setReplication, setOwner, setTimes, setWorkingDirectory, 1361 * setPermission, getFileChecksum, setVerifyChecksum, getFileBlockLocations, 1362 * getFsStatus, getFileStatus, exists, and listStatus. 1363 * 1364 * Symlink targets are stored as given to createSymlink, assuming the 1365 * underlying file system is capable of storing a fully qualified URI. 1366 * Dangling symlinks are permitted. FileContext supports four types of 1367 * symlink targets, and resolves them as follows 1368 * <pre> 1369 * Given a path referring to a symlink of form: 1370 * 1371 * <---X---> 1372 * fs://host/A/B/link 1373 * <-----Y-----> 1374 * 1375 * In this path X is the scheme and authority that identify the file system, 1376 * and Y is the path leading up to the final path component "link". If Y is 1377 * a symlink itself then let Y' be the target of Y and X' be the scheme and 1378 * authority of Y'. Symlink targets may: 1379 * 1380 * 1. Fully qualified URIs 1381 * 1382 * fs://hostX/A/B/file Resolved according to the target file system. 1383 * 1384 * 2. Partially qualified URIs (eg scheme but no host) 1385 * 1386 * fs:///A/B/file Resolved according to the target file system. Eg resolving 1387 * a symlink to hdfs:///A results in an exception because 1388 * HDFS URIs must be fully qualified, while a symlink to 1389 * file:///A will not since Hadoop's local file systems 1390 * require partially qualified URIs. 1391 * 1392 * 3. Relative paths 1393 * 1394 * path Resolves to [Y'][path]. Eg if Y resolves to hdfs://host/A and path 1395 * is "../B/file" then [Y'][path] is hdfs://host/B/file 1396 * 1397 * 4. Absolute paths 1398 * 1399 * path Resolves to [X'][path]. Eg if Y resolves hdfs://host/A/B and path 1400 * is "/file" then [X][path] is hdfs://host/file 1401 * </pre> 1402 * 1403 * @param target the target of the symbolic link 1404 * @param link the path to be created that points to target 1405 * @param createParent if true then missing parent dirs are created if 1406 * false then parent must exist 1407 * 1408 * 1409 * @throws AccessControlException If access is denied 1410 * @throws FileAlreadyExistsException If file <code>linkcode> already exists 1411 * @throws FileNotFoundException If <code>target</code> does not exist 1412 * @throws ParentNotDirectoryException If parent of <code>link</code> is not a 1413 * directory. 1414 * @throws UnsupportedFileSystemException If file system for 1415 * <code>target</code> or <code>link</code> is not supported 1416 * @throws IOException If an I/O error occurred 1417 */ 1418 @SuppressWarnings("deprecation") 1419 public void createSymlink(final Path target, final Path link, 1420 final boolean createParent) throws AccessControlException, 1421 FileAlreadyExistsException, FileNotFoundException, 1422 ParentNotDirectoryException, UnsupportedFileSystemException, 1423 IOException { 1424 if (!FileSystem.areSymlinksEnabled()) { 1425 throw new UnsupportedOperationException("Symlinks not supported"); 1426 } 1427 final Path nonRelLink = fixRelativePart(link); 1428 new FSLinkResolver<Void>() { 1429 @Override 1430 public Void next(final AbstractFileSystem fs, final Path p) 1431 throws IOException, UnresolvedLinkException { 1432 fs.createSymlink(target, p, createParent); 1433 return null; 1434 } 1435 }.resolve(this, nonRelLink); 1436 } 1437 1438 /** 1439 * List the statuses of the files/directories in the given path if the path is 1440 * a directory. 1441 * 1442 * @param f is the path 1443 * 1444 * @return an iterator that traverses statuses of the files/directories 1445 * in the given path 1446 * 1447 * @throws AccessControlException If access is denied 1448 * @throws FileNotFoundException If <code>f</code> does not exist 1449 * @throws UnsupportedFileSystemException If file system for <code>f</code> is 1450 * not supported 1451 * @throws IOException If an I/O error occurred 1452 * 1453 * Exceptions applicable to file systems accessed over RPC: 1454 * @throws RpcClientException If an exception occurred in the RPC client 1455 * @throws RpcServerException If an exception occurred in the RPC server 1456 * @throws UnexpectedServerException If server implementation throws 1457 * undeclared exception to RPC server 1458 */ 1459 public RemoteIterator<FileStatus> listStatus(final Path f) throws 1460 AccessControlException, FileNotFoundException, 1461 UnsupportedFileSystemException, IOException { 1462 final Path absF = fixRelativePart(f); 1463 return new FSLinkResolver<RemoteIterator<FileStatus>>() { 1464 @Override 1465 public RemoteIterator<FileStatus> next( 1466 final AbstractFileSystem fs, final Path p) 1467 throws IOException, UnresolvedLinkException { 1468 return fs.listStatusIterator(p); 1469 } 1470 }.resolve(this, absF); 1471 } 1472 1473 /** 1474 * @return an iterator over the corrupt files under the given path 1475 * (may contain duplicates if a file has more than one corrupt block) 1476 * @throws IOException 1477 */ 1478 public RemoteIterator<Path> listCorruptFileBlocks(Path path) 1479 throws IOException { 1480 final Path absF = fixRelativePart(path); 1481 return new FSLinkResolver<RemoteIterator<Path>>() { 1482 @Override 1483 public RemoteIterator<Path> next(final AbstractFileSystem fs, 1484 final Path p) 1485 throws IOException, UnresolvedLinkException { 1486 return fs.listCorruptFileBlocks(p); 1487 } 1488 }.resolve(this, absF); 1489 } 1490 1491 /** 1492 * List the statuses of the files/directories in the given path if the path is 1493 * a directory. 1494 * Return the file's status and block locations If the path is a file. 1495 * 1496 * If a returned status is a file, it contains the file's block locations. 1497 * 1498 * @param f is the path 1499 * 1500 * @return an iterator that traverses statuses of the files/directories 1501 * in the given path 1502 * If any IO exception (for example the input directory gets deleted while 1503 * listing is being executed), next() or hasNext() of the returned iterator 1504 * may throw a RuntimeException with the io exception as the cause. 1505 * 1506 * @throws AccessControlException If access is denied 1507 * @throws FileNotFoundException If <code>f</code> does not exist 1508 * @throws UnsupportedFileSystemException If file system for <code>f</code> is 1509 * not supported 1510 * @throws IOException If an I/O error occurred 1511 * 1512 * Exceptions applicable to file systems accessed over RPC: 1513 * @throws RpcClientException If an exception occurred in the RPC client 1514 * @throws RpcServerException If an exception occurred in the RPC server 1515 * @throws UnexpectedServerException If server implementation throws 1516 * undeclared exception to RPC server 1517 */ 1518 public RemoteIterator<LocatedFileStatus> listLocatedStatus( 1519 final Path f) throws 1520 AccessControlException, FileNotFoundException, 1521 UnsupportedFileSystemException, IOException { 1522 final Path absF = fixRelativePart(f); 1523 return new FSLinkResolver<RemoteIterator<LocatedFileStatus>>() { 1524 @Override 1525 public RemoteIterator<LocatedFileStatus> next( 1526 final AbstractFileSystem fs, final Path p) 1527 throws IOException, UnresolvedLinkException { 1528 return fs.listLocatedStatus(p); 1529 } 1530 }.resolve(this, absF); 1531 } 1532 1533 /** 1534 * Mark a path to be deleted on JVM shutdown. 1535 * 1536 * @param f the existing path to delete. 1537 * 1538 * @return true if deleteOnExit is successful, otherwise false. 1539 * 1540 * @throws AccessControlException If access is denied 1541 * @throws UnsupportedFileSystemException If file system for <code>f</code> is 1542 * not supported 1543 * @throws IOException If an I/O error occurred 1544 * 1545 * Exceptions applicable to file systems accessed over RPC: 1546 * @throws RpcClientException If an exception occurred in the RPC client 1547 * @throws RpcServerException If an exception occurred in the RPC server 1548 * @throws UnexpectedServerException If server implementation throws 1549 * undeclared exception to RPC server 1550 */ 1551 public boolean deleteOnExit(Path f) throws AccessControlException, 1552 IOException { 1553 if (!this.util().exists(f)) { 1554 return false; 1555 } 1556 synchronized (DELETE_ON_EXIT) { 1557 if (DELETE_ON_EXIT.isEmpty()) { 1558 ShutdownHookManager.get().addShutdownHook(FINALIZER, SHUTDOWN_HOOK_PRIORITY); 1559 } 1560 1561 Set<Path> set = DELETE_ON_EXIT.get(this); 1562 if (set == null) { 1563 set = new TreeSet<Path>(); 1564 DELETE_ON_EXIT.put(this, set); 1565 } 1566 set.add(f); 1567 } 1568 return true; 1569 } 1570 1571 private final Util util; 1572 public Util util() { 1573 return util; 1574 } 1575 1576 1577 /** 1578 * Utility/library methods built over the basic FileContext methods. 1579 * Since this are library functions, the oprtation are not atomic 1580 * and some of them may partially complete if other threads are making 1581 * changes to the same part of the name space. 1582 */ 1583 public class Util { 1584 /** 1585 * Does the file exist? 1586 * Note: Avoid using this method if you already have FileStatus in hand. 1587 * Instead reuse the FileStatus 1588 * @param f the file or dir to be checked 1589 * 1590 * @throws AccessControlException If access is denied 1591 * @throws IOException If an I/O error occurred 1592 * @throws UnsupportedFileSystemException If file system for <code>f</code> is 1593 * not supported 1594 * 1595 * Exceptions applicable to file systems accessed over RPC: 1596 * @throws RpcClientException If an exception occurred in the RPC client 1597 * @throws RpcServerException If an exception occurred in the RPC server 1598 * @throws UnexpectedServerException If server implementation throws 1599 * undeclared exception to RPC server 1600 */ 1601 public boolean exists(final Path f) throws AccessControlException, 1602 UnsupportedFileSystemException, IOException { 1603 try { 1604 FileStatus fs = FileContext.this.getFileStatus(f); 1605 assert fs != null; 1606 return true; 1607 } catch (FileNotFoundException e) { 1608 return false; 1609 } 1610 } 1611 1612 /** 1613 * Return the {@link ContentSummary} of path f. 1614 * @param f path 1615 * 1616 * @return the {@link ContentSummary} of path f. 1617 * 1618 * @throws AccessControlException If access is denied 1619 * @throws FileNotFoundException If <code>f</code> does not exist 1620 * @throws UnsupportedFileSystemException If file system for 1621 * <code>f</code> is not supported 1622 * @throws IOException If an I/O error occurred 1623 * 1624 * Exceptions applicable to file systems accessed over RPC: 1625 * @throws RpcClientException If an exception occurred in the RPC client 1626 * @throws RpcServerException If an exception occurred in the RPC server 1627 * @throws UnexpectedServerException If server implementation throws 1628 * undeclared exception to RPC server 1629 */ 1630 public ContentSummary getContentSummary(Path f) 1631 throws AccessControlException, FileNotFoundException, 1632 UnsupportedFileSystemException, IOException { 1633 FileStatus status = FileContext.this.getFileStatus(f); 1634 if (status.isFile()) { 1635 return new ContentSummary(status.getLen(), 1, 0); 1636 } 1637 long[] summary = {0, 0, 1}; 1638 RemoteIterator<FileStatus> statusIterator = 1639 FileContext.this.listStatus(f); 1640 while(statusIterator.hasNext()) { 1641 FileStatus s = statusIterator.next(); 1642 ContentSummary c = s.isDirectory() ? getContentSummary(s.getPath()) : 1643 new ContentSummary(s.getLen(), 1, 0); 1644 summary[0] += c.getLength(); 1645 summary[1] += c.getFileCount(); 1646 summary[2] += c.getDirectoryCount(); 1647 } 1648 return new ContentSummary(summary[0], summary[1], summary[2]); 1649 } 1650 1651 /** 1652 * See {@link #listStatus(Path[], PathFilter)} 1653 */ 1654 public FileStatus[] listStatus(Path[] files) throws AccessControlException, 1655 FileNotFoundException, IOException { 1656 return listStatus(files, DEFAULT_FILTER); 1657 } 1658 1659 /** 1660 * Filter files/directories in the given path using the user-supplied path 1661 * filter. 1662 * 1663 * @param f is the path name 1664 * @param filter is the user-supplied path filter 1665 * 1666 * @return an array of FileStatus objects for the files under the given path 1667 * after applying the filter 1668 * 1669 * @throws AccessControlException If access is denied 1670 * @throws FileNotFoundException If <code>f</code> does not exist 1671 * @throws UnsupportedFileSystemException If file system for 1672 * <code>pathPattern</code> is not supported 1673 * @throws IOException If an I/O error occurred 1674 * 1675 * Exceptions applicable to file systems accessed over RPC: 1676 * @throws RpcClientException If an exception occurred in the RPC client 1677 * @throws RpcServerException If an exception occurred in the RPC server 1678 * @throws UnexpectedServerException If server implementation throws 1679 * undeclared exception to RPC server 1680 */ 1681 public FileStatus[] listStatus(Path f, PathFilter filter) 1682 throws AccessControlException, FileNotFoundException, 1683 UnsupportedFileSystemException, IOException { 1684 ArrayList<FileStatus> results = new ArrayList<FileStatus>(); 1685 listStatus(results, f, filter); 1686 return results.toArray(new FileStatus[results.size()]); 1687 } 1688 1689 /** 1690 * Filter files/directories in the given list of paths using user-supplied 1691 * path filter. 1692 * 1693 * @param files is a list of paths 1694 * @param filter is the filter 1695 * 1696 * @return a list of statuses for the files under the given paths after 1697 * applying the filter 1698 * 1699 * @throws AccessControlException If access is denied 1700 * @throws FileNotFoundException If a file in <code>files</code> does not 1701 * exist 1702 * @throws IOException If an I/O error occurred 1703 * 1704 * Exceptions applicable to file systems accessed over RPC: 1705 * @throws RpcClientException If an exception occurred in the RPC client 1706 * @throws RpcServerException If an exception occurred in the RPC server 1707 * @throws UnexpectedServerException If server implementation throws 1708 * undeclared exception to RPC server 1709 */ 1710 public FileStatus[] listStatus(Path[] files, PathFilter filter) 1711 throws AccessControlException, FileNotFoundException, IOException { 1712 ArrayList<FileStatus> results = new ArrayList<FileStatus>(); 1713 for (int i = 0; i < files.length; i++) { 1714 listStatus(results, files[i], filter); 1715 } 1716 return results.toArray(new FileStatus[results.size()]); 1717 } 1718 1719 /* 1720 * Filter files/directories in the given path using the user-supplied path 1721 * filter. Results are added to the given array <code>results</code>. 1722 */ 1723 private void listStatus(ArrayList<FileStatus> results, Path f, 1724 PathFilter filter) throws AccessControlException, 1725 FileNotFoundException, IOException { 1726 FileStatus[] listing = listStatus(f); 1727 if (listing != null) { 1728 for (int i = 0; i < listing.length; i++) { 1729 if (filter.accept(listing[i].getPath())) { 1730 results.add(listing[i]); 1731 } 1732 } 1733 } 1734 } 1735 1736 /** 1737 * List the statuses of the files/directories in the given path 1738 * if the path is a directory. 1739 * 1740 * @param f is the path 1741 * 1742 * @return an array that contains statuses of the files/directories 1743 * in the given path 1744 * 1745 * @throws AccessControlException If access is denied 1746 * @throws FileNotFoundException If <code>f</code> does not exist 1747 * @throws UnsupportedFileSystemException If file system for <code>f</code> is 1748 * not supported 1749 * @throws IOException If an I/O error occurred 1750 * 1751 * Exceptions applicable to file systems accessed over RPC: 1752 * @throws RpcClientException If an exception occurred in the RPC client 1753 * @throws RpcServerException If an exception occurred in the RPC server 1754 * @throws UnexpectedServerException If server implementation throws 1755 * undeclared exception to RPC server 1756 */ 1757 public FileStatus[] listStatus(final Path f) throws AccessControlException, 1758 FileNotFoundException, UnsupportedFileSystemException, 1759 IOException { 1760 final Path absF = fixRelativePart(f); 1761 return new FSLinkResolver<FileStatus[]>() { 1762 @Override 1763 public FileStatus[] next(final AbstractFileSystem fs, final Path p) 1764 throws IOException, UnresolvedLinkException { 1765 return fs.listStatus(p); 1766 } 1767 }.resolve(FileContext.this, absF); 1768 } 1769 1770 /** 1771 * List the statuses and block locations of the files in the given path. 1772 * 1773 * If the path is a directory, 1774 * if recursive is false, returns files in the directory; 1775 * if recursive is true, return files in the subtree rooted at the path. 1776 * The subtree is traversed in the depth-first order. 1777 * If the path is a file, return the file's status and block locations. 1778 * Files across symbolic links are also returned. 1779 * 1780 * @param f is the path 1781 * @param recursive if the subdirectories need to be traversed recursively 1782 * 1783 * @return an iterator that traverses statuses of the files 1784 * If any IO exception (for example a sub-directory gets deleted while 1785 * listing is being executed), next() or hasNext() of the returned iterator 1786 * may throw a RuntimeException with the IO exception as the cause. 1787 * 1788 * @throws AccessControlException If access is denied 1789 * @throws FileNotFoundException If <code>f</code> does not exist 1790 * @throws UnsupportedFileSystemException If file system for <code>f</code> 1791 * is not supported 1792 * @throws IOException If an I/O error occurred 1793 * 1794 * Exceptions applicable to file systems accessed over RPC: 1795 * @throws RpcClientException If an exception occurred in the RPC client 1796 * @throws RpcServerException If an exception occurred in the RPC server 1797 * @throws UnexpectedServerException If server implementation throws 1798 * undeclared exception to RPC server 1799 */ 1800 public RemoteIterator<LocatedFileStatus> listFiles( 1801 final Path f, final boolean recursive) throws AccessControlException, 1802 FileNotFoundException, UnsupportedFileSystemException, 1803 IOException { 1804 return new RemoteIterator<LocatedFileStatus>() { 1805 private Stack<RemoteIterator<LocatedFileStatus>> itors = 1806 new Stack<RemoteIterator<LocatedFileStatus>>(); 1807 RemoteIterator<LocatedFileStatus> curItor = listLocatedStatus(f); 1808 LocatedFileStatus curFile; 1809 1810 /** 1811 * Returns <tt>true</tt> if the iterator has more files. 1812 * 1813 * @return <tt>true</tt> if the iterator has more files. 1814 * @throws AccessControlException if not allowed to access next 1815 * file's status or locations 1816 * @throws FileNotFoundException if next file does not exist any more 1817 * @throws UnsupportedFileSystemException if next file's 1818 * fs is unsupported 1819 * @throws IOException for all other IO errors 1820 * for example, NameNode is not avaialbe or 1821 * NameNode throws IOException due to an error 1822 * while getting the status or block locations 1823 */ 1824 @Override 1825 public boolean hasNext() throws IOException { 1826 while (curFile == null) { 1827 if (curItor.hasNext()) { 1828 handleFileStat(curItor.next()); 1829 } else if (!itors.empty()) { 1830 curItor = itors.pop(); 1831 } else { 1832 return false; 1833 } 1834 } 1835 return true; 1836 } 1837 1838 /** 1839 * Process the input stat. 1840 * If it is a file, return the file stat. 1841 * If it is a directory, traverse the directory if recursive is true; 1842 * ignore it if recursive is false. 1843 * If it is a symlink, resolve the symlink first and then process it 1844 * depending on if it is a file or directory. 1845 * @param stat input status 1846 * @throws AccessControlException if access is denied 1847 * @throws FileNotFoundException if file is not found 1848 * @throws UnsupportedFileSystemException if fs is not supported 1849 * @throws IOException for all other IO errors 1850 */ 1851 private void handleFileStat(LocatedFileStatus stat) 1852 throws IOException { 1853 if (stat.isFile()) { // file 1854 curFile = stat; 1855 } else if (stat.isSymlink()) { // symbolic link 1856 // resolve symbolic link 1857 FileStatus symstat = FileContext.this.getFileStatus( 1858 stat.getSymlink()); 1859 if (symstat.isFile() || (recursive && symstat.isDirectory())) { 1860 itors.push(curItor); 1861 curItor = listLocatedStatus(stat.getPath()); 1862 } 1863 } else if (recursive) { // directory 1864 itors.push(curItor); 1865 curItor = listLocatedStatus(stat.getPath()); 1866 } 1867 } 1868 1869 /** 1870 * Returns the next file's status with its block locations 1871 * 1872 * @throws AccessControlException if not allowed to access next 1873 * file's status or locations 1874 * @throws FileNotFoundException if next file does not exist any more 1875 * @throws UnsupportedFileSystemException if next file's 1876 * fs is unsupported 1877 * @throws IOException for all other IO errors 1878 * for example, NameNode is not avaialbe or 1879 * NameNode throws IOException due to an error 1880 * while getting the status or block locations 1881 */ 1882 @Override 1883 public LocatedFileStatus next() throws IOException { 1884 if (hasNext()) { 1885 LocatedFileStatus result = curFile; 1886 curFile = null; 1887 return result; 1888 } 1889 throw new java.util.NoSuchElementException("No more entry in " + f); 1890 } 1891 }; 1892 } 1893 1894 /** 1895 * <p>Return all the files that match filePattern and are not checksum 1896 * files. Results are sorted by their names. 1897 * 1898 * <p> 1899 * A filename pattern is composed of <i>regular</i> characters and 1900 * <i>special pattern matching</i> characters, which are: 1901 * 1902 * <dl> 1903 * <dd> 1904 * <dl> 1905 * <p> 1906 * <dt> <tt> ? </tt> 1907 * <dd> Matches any single character. 1908 * 1909 * <p> 1910 * <dt> <tt> * </tt> 1911 * <dd> Matches zero or more characters. 1912 * 1913 * <p> 1914 * <dt> <tt> [<i>abc</i>] </tt> 1915 * <dd> Matches a single character from character set 1916 * <tt>{<i>a,b,c</i>}</tt>. 1917 * 1918 * <p> 1919 * <dt> <tt> [<i>a</i>-<i>b</i>] </tt> 1920 * <dd> Matches a single character from the character range 1921 * <tt>{<i>a...b</i>}</tt>. Note: character <tt><i>a</i></tt> must be 1922 * lexicographically less than or equal to character <tt><i>b</i></tt>. 1923 * 1924 * <p> 1925 * <dt> <tt> [^<i>a</i>] </tt> 1926 * <dd> Matches a single char that is not from character set or range 1927 * <tt>{<i>a</i>}</tt>. Note that the <tt>^</tt> character must occur 1928 * immediately to the right of the opening bracket. 1929 * 1930 * <p> 1931 * <dt> <tt> \<i>c</i> </tt> 1932 * <dd> Removes (escapes) any special meaning of character <i>c</i>. 1933 * 1934 * <p> 1935 * <dt> <tt> {ab,cd} </tt> 1936 * <dd> Matches a string from the string set <tt>{<i>ab, cd</i>} </tt> 1937 * 1938 * <p> 1939 * <dt> <tt> {ab,c{de,fh}} </tt> 1940 * <dd> Matches a string from string set <tt>{<i>ab, cde, cfh</i>}</tt> 1941 * 1942 * </dl> 1943 * </dd> 1944 * </dl> 1945 * 1946 * @param pathPattern a regular expression specifying a pth pattern 1947 * 1948 * @return an array of paths that match the path pattern 1949 * 1950 * @throws AccessControlException If access is denied 1951 * @throws UnsupportedFileSystemException If file system for 1952 * <code>pathPattern</code> is not supported 1953 * @throws IOException If an I/O error occurred 1954 * 1955 * Exceptions applicable to file systems accessed over RPC: 1956 * @throws RpcClientException If an exception occurred in the RPC client 1957 * @throws RpcServerException If an exception occurred in the RPC server 1958 * @throws UnexpectedServerException If server implementation throws 1959 * undeclared exception to RPC server 1960 */ 1961 public FileStatus[] globStatus(Path pathPattern) 1962 throws AccessControlException, UnsupportedFileSystemException, 1963 IOException { 1964 return new Globber(FileContext.this, pathPattern, DEFAULT_FILTER).glob(); 1965 } 1966 1967 /** 1968 * Return an array of FileStatus objects whose path names match pathPattern 1969 * and is accepted by the user-supplied path filter. Results are sorted by 1970 * their path names. 1971 * Return null if pathPattern has no glob and the path does not exist. 1972 * Return an empty array if pathPattern has a glob and no path matches it. 1973 * 1974 * @param pathPattern regular expression specifying the path pattern 1975 * @param filter user-supplied path filter 1976 * 1977 * @return an array of FileStatus objects 1978 * 1979 * @throws AccessControlException If access is denied 1980 * @throws UnsupportedFileSystemException If file system for 1981 * <code>pathPattern</code> is not supported 1982 * @throws IOException If an I/O error occurred 1983 * 1984 * Exceptions applicable to file systems accessed over RPC: 1985 * @throws RpcClientException If an exception occurred in the RPC client 1986 * @throws RpcServerException If an exception occurred in the RPC server 1987 * @throws UnexpectedServerException If server implementation throws 1988 * undeclared exception to RPC server 1989 */ 1990 public FileStatus[] globStatus(final Path pathPattern, 1991 final PathFilter filter) throws AccessControlException, 1992 UnsupportedFileSystemException, IOException { 1993 return new Globber(FileContext.this, pathPattern, filter).glob(); 1994 } 1995 1996 /** 1997 * Copy file from src to dest. See 1998 * {@link #copy(Path, Path, boolean, boolean)} 1999 */ 2000 public boolean copy(final Path src, final Path dst) 2001 throws AccessControlException, FileAlreadyExistsException, 2002 FileNotFoundException, ParentNotDirectoryException, 2003 UnsupportedFileSystemException, IOException { 2004 return copy(src, dst, false, false); 2005 } 2006 2007 /** 2008 * Copy from src to dst, optionally deleting src and overwriting dst. 2009 * @param src 2010 * @param dst 2011 * @param deleteSource - delete src if true 2012 * @param overwrite overwrite dst if true; throw IOException if dst exists 2013 * and overwrite is false. 2014 * 2015 * @return true if copy is successful 2016 * 2017 * @throws AccessControlException If access is denied 2018 * @throws FileAlreadyExistsException If <code>dst</code> already exists 2019 * @throws FileNotFoundException If <code>src</code> does not exist 2020 * @throws ParentNotDirectoryException If parent of <code>dst</code> is not 2021 * a directory 2022 * @throws UnsupportedFileSystemException If file system for 2023 * <code>src</code> or <code>dst</code> is not supported 2024 * @throws IOException If an I/O error occurred 2025 * 2026 * Exceptions applicable to file systems accessed over RPC: 2027 * @throws RpcClientException If an exception occurred in the RPC client 2028 * @throws RpcServerException If an exception occurred in the RPC server 2029 * @throws UnexpectedServerException If server implementation throws 2030 * undeclared exception to RPC server 2031 * 2032 * RuntimeExceptions: 2033 * @throws InvalidPathException If path <code>dst</code> is invalid 2034 */ 2035 public boolean copy(final Path src, final Path dst, boolean deleteSource, 2036 boolean overwrite) throws AccessControlException, 2037 FileAlreadyExistsException, FileNotFoundException, 2038 ParentNotDirectoryException, UnsupportedFileSystemException, 2039 IOException { 2040 src.checkNotSchemeWithRelative(); 2041 dst.checkNotSchemeWithRelative(); 2042 Path qSrc = makeQualified(src); 2043 Path qDst = makeQualified(dst); 2044 checkDest(qSrc.getName(), qDst, overwrite); 2045 FileStatus fs = FileContext.this.getFileStatus(qSrc); 2046 if (fs.isDirectory()) { 2047 checkDependencies(qSrc, qDst); 2048 mkdir(qDst, FsPermission.getDirDefault(), true); 2049 FileStatus[] contents = listStatus(qSrc); 2050 for (FileStatus content : contents) { 2051 copy(makeQualified(content.getPath()), makeQualified(new Path(qDst, 2052 content.getPath().getName())), deleteSource, overwrite); 2053 } 2054 } else { 2055 InputStream in=null; 2056 OutputStream out = null; 2057 try { 2058 in = open(qSrc); 2059 EnumSet<CreateFlag> createFlag = overwrite ? EnumSet.of( 2060 CreateFlag.CREATE, CreateFlag.OVERWRITE) : 2061 EnumSet.of(CreateFlag.CREATE); 2062 out = create(qDst, createFlag); 2063 IOUtils.copyBytes(in, out, conf, true); 2064 } finally { 2065 IOUtils.closeStream(out); 2066 IOUtils.closeStream(in); 2067 } 2068 } 2069 if (deleteSource) { 2070 return delete(qSrc, true); 2071 } else { 2072 return true; 2073 } 2074 } 2075 } 2076 2077 /** 2078 * Check if copying srcName to dst would overwrite an existing 2079 * file or directory. 2080 * @param srcName File or directory to be copied. 2081 * @param dst Destination to copy srcName to. 2082 * @param overwrite Whether it's ok to overwrite an existing file. 2083 * @throws AccessControlException If access is denied. 2084 * @throws IOException If dst is an existing directory, or dst is an 2085 * existing file and the overwrite option is not passed. 2086 */ 2087 private void checkDest(String srcName, Path dst, boolean overwrite) 2088 throws AccessControlException, IOException { 2089 try { 2090 FileStatus dstFs = getFileStatus(dst); 2091 if (dstFs.isDirectory()) { 2092 if (null == srcName) { 2093 throw new IOException("Target " + dst + " is a directory"); 2094 } 2095 // Recurse to check if dst/srcName exists. 2096 checkDest(null, new Path(dst, srcName), overwrite); 2097 } else if (!overwrite) { 2098 throw new IOException("Target " + new Path(dst, srcName) 2099 + " already exists"); 2100 } 2101 } catch (FileNotFoundException e) { 2102 // dst does not exist - OK to copy. 2103 } 2104 } 2105 2106 // 2107 // If the destination is a subdirectory of the source, then 2108 // generate exception 2109 // 2110 private static void checkDependencies(Path qualSrc, Path qualDst) 2111 throws IOException { 2112 if (isSameFS(qualSrc, qualDst)) { 2113 String srcq = qualSrc.toString() + Path.SEPARATOR; 2114 String dstq = qualDst.toString() + Path.SEPARATOR; 2115 if (dstq.startsWith(srcq)) { 2116 if (srcq.length() == dstq.length()) { 2117 throw new IOException("Cannot copy " + qualSrc + " to itself."); 2118 } else { 2119 throw new IOException("Cannot copy " + qualSrc + 2120 " to its subdirectory " + qualDst); 2121 } 2122 } 2123 } 2124 } 2125 2126 /** 2127 * Are qualSrc and qualDst of the same file system? 2128 * @param qualPath1 - fully qualified path 2129 * @param qualPath2 - fully qualified path 2130 * @return 2131 */ 2132 private static boolean isSameFS(Path qualPath1, Path qualPath2) { 2133 URI srcUri = qualPath1.toUri(); 2134 URI dstUri = qualPath2.toUri(); 2135 return (srcUri.getScheme().equals(dstUri.getScheme()) && 2136 !(srcUri.getAuthority() != null && dstUri.getAuthority() != null && srcUri 2137 .getAuthority().equals(dstUri.getAuthority()))); 2138 } 2139 2140 /** 2141 * Deletes all the paths in deleteOnExit on JVM shutdown. 2142 */ 2143 static class FileContextFinalizer implements Runnable { 2144 @Override 2145 public synchronized void run() { 2146 processDeleteOnExit(); 2147 } 2148 } 2149 2150 /** 2151 * Resolves all symbolic links in the specified path. 2152 * Returns the new path object. 2153 */ 2154 protected Path resolve(final Path f) throws FileNotFoundException, 2155 UnresolvedLinkException, AccessControlException, IOException { 2156 return new FSLinkResolver<Path>() { 2157 @Override 2158 public Path next(final AbstractFileSystem fs, final Path p) 2159 throws IOException, UnresolvedLinkException { 2160 return fs.resolvePath(p); 2161 } 2162 }.resolve(this, f); 2163 } 2164 2165 /** 2166 * Resolves all symbolic links in the specified path leading up 2167 * to, but not including the final path component. 2168 * @param f path to resolve 2169 * @return the new path object. 2170 */ 2171 protected Path resolveIntermediate(final Path f) throws IOException { 2172 return new FSLinkResolver<FileStatus>() { 2173 @Override 2174 public FileStatus next(final AbstractFileSystem fs, final Path p) 2175 throws IOException, UnresolvedLinkException { 2176 return fs.getFileLinkStatus(p); 2177 } 2178 }.resolve(this, f).getPath(); 2179 } 2180 2181 /** 2182 * Returns the list of AbstractFileSystems accessed in the path. The list may 2183 * contain more than one AbstractFileSystems objects in case of symlinks. 2184 * 2185 * @param f 2186 * Path which needs to be resolved 2187 * @return List of AbstractFileSystems accessed in the path 2188 * @throws IOException 2189 */ 2190 Set<AbstractFileSystem> resolveAbstractFileSystems(final Path f) 2191 throws IOException { 2192 final Path absF = fixRelativePart(f); 2193 final HashSet<AbstractFileSystem> result 2194 = new HashSet<AbstractFileSystem>(); 2195 new FSLinkResolver<Void>() { 2196 @Override 2197 public Void next(final AbstractFileSystem fs, final Path p) 2198 throws IOException, UnresolvedLinkException { 2199 result.add(fs); 2200 fs.getFileStatus(p); 2201 return null; 2202 } 2203 }.resolve(this, absF); 2204 return result; 2205 } 2206 2207 /** 2208 * Get the statistics for a particular file system 2209 * 2210 * @param uri 2211 * the uri to lookup the statistics. Only scheme and authority part 2212 * of the uri are used as the key to store and lookup. 2213 * @return a statistics object 2214 */ 2215 public static Statistics getStatistics(URI uri) { 2216 return AbstractFileSystem.getStatistics(uri); 2217 } 2218 2219 /** 2220 * Clears all the statistics stored in AbstractFileSystem, for all the file 2221 * systems. 2222 */ 2223 public static void clearStatistics() { 2224 AbstractFileSystem.clearStatistics(); 2225 } 2226 2227 /** 2228 * Prints the statistics to standard output. File System is identified by the 2229 * scheme and authority. 2230 */ 2231 public static void printStatistics() { 2232 AbstractFileSystem.printStatistics(); 2233 } 2234 2235 /** 2236 * @return Map of uri and statistics for each filesystem instantiated. The uri 2237 * consists of scheme and authority for the filesystem. 2238 */ 2239 public static Map<URI, Statistics> getAllStatistics() { 2240 return AbstractFileSystem.getAllStatistics(); 2241 } 2242 2243 /** 2244 * Get delegation tokens for the file systems accessed for a given 2245 * path. 2246 * @param p Path for which delegations tokens are requested. 2247 * @param renewer the account name that is allowed to renew the token. 2248 * @return List of delegation tokens. 2249 * @throws IOException 2250 */ 2251 @InterfaceAudience.LimitedPrivate( { "HDFS", "MapReduce" }) 2252 public List<Token<?>> getDelegationTokens( 2253 Path p, String renewer) throws IOException { 2254 Set<AbstractFileSystem> afsSet = resolveAbstractFileSystems(p); 2255 List<Token<?>> tokenList = 2256 new ArrayList<Token<?>>(); 2257 for (AbstractFileSystem afs : afsSet) { 2258 List<Token<?>> afsTokens = afs.getDelegationTokens(renewer); 2259 tokenList.addAll(afsTokens); 2260 } 2261 return tokenList; 2262 } 2263 2264 /** 2265 * Modifies ACL entries of files and directories. This method can add new ACL 2266 * entries or modify the permissions on existing ACL entries. All existing 2267 * ACL entries that are not specified in this call are retained without 2268 * changes. (Modifications are merged into the current ACL.) 2269 * 2270 * @param path Path to modify 2271 * @param aclSpec List<AclEntry> describing modifications 2272 * @throws IOException if an ACL could not be modified 2273 */ 2274 public void modifyAclEntries(final Path path, final List<AclEntry> aclSpec) 2275 throws IOException { 2276 Path absF = fixRelativePart(path); 2277 new FSLinkResolver<Void>() { 2278 @Override 2279 public Void next(final AbstractFileSystem fs, final Path p) 2280 throws IOException { 2281 fs.modifyAclEntries(p, aclSpec); 2282 return null; 2283 } 2284 }.resolve(this, absF); 2285 } 2286 2287 /** 2288 * Removes ACL entries from files and directories. Other ACL entries are 2289 * retained. 2290 * 2291 * @param path Path to modify 2292 * @param aclSpec List<AclEntry> describing entries to remove 2293 * @throws IOException if an ACL could not be modified 2294 */ 2295 public void removeAclEntries(final Path path, final List<AclEntry> aclSpec) 2296 throws IOException { 2297 Path absF = fixRelativePart(path); 2298 new FSLinkResolver<Void>() { 2299 @Override 2300 public Void next(final AbstractFileSystem fs, final Path p) 2301 throws IOException { 2302 fs.removeAclEntries(p, aclSpec); 2303 return null; 2304 } 2305 }.resolve(this, absF); 2306 } 2307 2308 /** 2309 * Removes all default ACL entries from files and directories. 2310 * 2311 * @param path Path to modify 2312 * @throws IOException if an ACL could not be modified 2313 */ 2314 public void removeDefaultAcl(Path path) 2315 throws IOException { 2316 Path absF = fixRelativePart(path); 2317 new FSLinkResolver<Void>() { 2318 @Override 2319 public Void next(final AbstractFileSystem fs, final Path p) 2320 throws IOException { 2321 fs.removeDefaultAcl(p); 2322 return null; 2323 } 2324 }.resolve(this, absF); 2325 } 2326 2327 /** 2328 * Removes all but the base ACL entries of files and directories. The entries 2329 * for user, group, and others are retained for compatibility with permission 2330 * bits. 2331 * 2332 * @param path Path to modify 2333 * @throws IOException if an ACL could not be removed 2334 */ 2335 public void removeAcl(Path path) throws IOException { 2336 Path absF = fixRelativePart(path); 2337 new FSLinkResolver<Void>() { 2338 @Override 2339 public Void next(final AbstractFileSystem fs, final Path p) 2340 throws IOException { 2341 fs.removeAcl(p); 2342 return null; 2343 } 2344 }.resolve(this, absF); 2345 } 2346 2347 /** 2348 * Fully replaces ACL of files and directories, discarding all existing 2349 * entries. 2350 * 2351 * @param path Path to modify 2352 * @param aclSpec List<AclEntry> describing modifications, must include entries 2353 * for user, group, and others for compatibility with permission bits. 2354 * @throws IOException if an ACL could not be modified 2355 */ 2356 public void setAcl(Path path, final List<AclEntry> aclSpec) 2357 throws IOException { 2358 Path absF = fixRelativePart(path); 2359 new FSLinkResolver<Void>() { 2360 @Override 2361 public Void next(final AbstractFileSystem fs, final Path p) 2362 throws IOException { 2363 fs.setAcl(p, aclSpec); 2364 return null; 2365 } 2366 }.resolve(this, absF); 2367 } 2368 2369 /** 2370 * Gets the ACLs of files and directories. 2371 * 2372 * @param path Path to get 2373 * @return RemoteIterator<AclStatus> which returns each AclStatus 2374 * @throws IOException if an ACL could not be read 2375 */ 2376 public AclStatus getAclStatus(Path path) throws IOException { 2377 Path absF = fixRelativePart(path); 2378 return new FSLinkResolver<AclStatus>() { 2379 @Override 2380 public AclStatus next(final AbstractFileSystem fs, final Path p) 2381 throws IOException { 2382 return fs.getAclStatus(p); 2383 } 2384 }.resolve(this, absF); 2385 } 2386 2387 /** 2388 * Set an xattr of a file or directory. 2389 * The name must be prefixed with the namespace followed by ".". For example, 2390 * "user.attr". 2391 * <p/> 2392 * Refer to the HDFS extended attributes user documentation for details. 2393 * 2394 * @param path Path to modify 2395 * @param name xattr name. 2396 * @param value xattr value. 2397 * @throws IOException 2398 */ 2399 public void setXAttr(Path path, String name, byte[] value) 2400 throws IOException { 2401 setXAttr(path, name, value, EnumSet.of(XAttrSetFlag.CREATE, 2402 XAttrSetFlag.REPLACE)); 2403 } 2404 2405 /** 2406 * Set an xattr of a file or directory. 2407 * The name must be prefixed with the namespace followed by ".". For example, 2408 * "user.attr". 2409 * <p/> 2410 * Refer to the HDFS extended attributes user documentation for details. 2411 * 2412 * @param path Path to modify 2413 * @param name xattr name. 2414 * @param value xattr value. 2415 * @param flag xattr set flag 2416 * @throws IOException 2417 */ 2418 public void setXAttr(Path path, final String name, final byte[] value, 2419 final EnumSet<XAttrSetFlag> flag) throws IOException { 2420 final Path absF = fixRelativePart(path); 2421 new FSLinkResolver<Void>() { 2422 @Override 2423 public Void next(final AbstractFileSystem fs, final Path p) 2424 throws IOException { 2425 fs.setXAttr(p, name, value, flag); 2426 return null; 2427 } 2428 }.resolve(this, absF); 2429 } 2430 2431 /** 2432 * Get an xattr for a file or directory. 2433 * The name must be prefixed with the namespace followed by ".". For example, 2434 * "user.attr". 2435 * <p/> 2436 * Refer to the HDFS extended attributes user documentation for details. 2437 * 2438 * @param path Path to get extended attribute 2439 * @param name xattr name. 2440 * @return byte[] xattr value. 2441 * @throws IOException 2442 */ 2443 public byte[] getXAttr(Path path, final String name) throws IOException { 2444 final Path absF = fixRelativePart(path); 2445 return new FSLinkResolver<byte[]>() { 2446 @Override 2447 public byte[] next(final AbstractFileSystem fs, final Path p) 2448 throws IOException { 2449 return fs.getXAttr(p, name); 2450 } 2451 }.resolve(this, absF); 2452 } 2453 2454 /** 2455 * Get all of the xattrs for a file or directory. 2456 * Only those xattrs for which the logged-in user has permissions to view 2457 * are returned. 2458 * <p/> 2459 * Refer to the HDFS extended attributes user documentation for details. 2460 * 2461 * @param path Path to get extended attributes 2462 * @return Map<String, byte[]> describing the XAttrs of the file or directory 2463 * @throws IOException 2464 */ 2465 public Map<String, byte[]> getXAttrs(Path path) throws IOException { 2466 final Path absF = fixRelativePart(path); 2467 return new FSLinkResolver<Map<String, byte[]>>() { 2468 @Override 2469 public Map<String, byte[]> next(final AbstractFileSystem fs, final Path p) 2470 throws IOException { 2471 return fs.getXAttrs(p); 2472 } 2473 }.resolve(this, absF); 2474 } 2475 2476 /** 2477 * Get all of the xattrs for a file or directory. 2478 * Only those xattrs for which the logged-in user has permissions to view 2479 * are returned. 2480 * <p/> 2481 * Refer to the HDFS extended attributes user documentation for details. 2482 * 2483 * @param path Path to get extended attributes 2484 * @param names XAttr names. 2485 * @return Map<String, byte[]> describing the XAttrs of the file or directory 2486 * @throws IOException 2487 */ 2488 public Map<String, byte[]> getXAttrs(Path path, final List<String> names) 2489 throws IOException { 2490 final Path absF = fixRelativePart(path); 2491 return new FSLinkResolver<Map<String, byte[]>>() { 2492 @Override 2493 public Map<String, byte[]> next(final AbstractFileSystem fs, final Path p) 2494 throws IOException { 2495 return fs.getXAttrs(p, names); 2496 } 2497 }.resolve(this, absF); 2498 } 2499 2500 /** 2501 * Remove an xattr of a file or directory. 2502 * The name must be prefixed with the namespace followed by ".". For example, 2503 * "user.attr". 2504 * <p/> 2505 * Refer to the HDFS extended attributes user documentation for details. 2506 * 2507 * @param path Path to remove extended attribute 2508 * @param name xattr name 2509 * @throws IOException 2510 */ 2511 public void removeXAttr(Path path, final String name) throws IOException { 2512 final Path absF = fixRelativePart(path); 2513 new FSLinkResolver<Void>() { 2514 @Override 2515 public Void next(final AbstractFileSystem fs, final Path p) 2516 throws IOException { 2517 fs.removeXAttr(p, name); 2518 return null; 2519 } 2520 }.resolve(this, absF); 2521 } 2522 2523 /** 2524 * Get all of the xattr names for a file or directory. 2525 * Only those xattr names which the logged-in user has permissions to view 2526 * are returned. 2527 * <p/> 2528 * Refer to the HDFS extended attributes user documentation for details. 2529 * 2530 * @param path Path to get extended attributes 2531 * @return List<String> of the XAttr names of the file or directory 2532 * @throws IOException 2533 */ 2534 public List<String> listXAttrs(Path path) throws IOException { 2535 final Path absF = fixRelativePart(path); 2536 return new FSLinkResolver<List<String>>() { 2537 @Override 2538 public List<String> next(final AbstractFileSystem fs, final Path p) 2539 throws IOException { 2540 return fs.listXAttrs(p); 2541 } 2542 }.resolve(this, absF); 2543 } 2544 2545 Tracer getTracer() { 2546 return tracer; 2547 } 2548}