public class TrackerDistributedCacheManager
extends java.lang.Object
Modifier and Type | Class and Description |
---|---|
protected class |
TrackerDistributedCacheManager.BaseDirManager
This class holds properties of each base directories and is responsible
for clean up unused cache files in base directories.
|
protected class |
TrackerDistributedCacheManager.CleanupThread
A thread to check and cleanup the unused files periodically
|
Modifier and Type | Field and Description |
---|---|
protected TrackerDistributedCacheManager.BaseDirManager |
baseDirManager |
protected TrackerDistributedCacheManager.CleanupThread |
cleanupThread |
static java.lang.String |
WORK_DIR_FIX |
Constructor and Description |
---|
TrackerDistributedCacheManager(org.apache.hadoop.conf.Configuration conf,
TaskController controller) |
TrackerDistributedCacheManager(org.apache.hadoop.conf.Configuration conf,
TaskController taskController,
MRAsyncDiskService asyncDiskService)
Creates a TrackerDistributedCacheManager with a MRAsyncDiskService.
|
Modifier and Type | Method and Description |
---|---|
static void |
createAllSymlink(org.apache.hadoop.conf.Configuration conf,
java.io.File jobCacheDir,
java.io.File workDir)
This method create symlinks for all files in a given dir in another
directory.
|
static void |
determineTimestampsAndCacheVisibilities(org.apache.hadoop.conf.Configuration job)
Determines timestamps of files to be cached, and stores those
in the configuration.
|
static long |
downloadCacheObject(org.apache.hadoop.conf.Configuration conf,
java.net.URI source,
org.apache.hadoop.fs.Path destination,
long desiredTimestamp,
boolean isArchive,
org.apache.hadoop.fs.permission.FsPermission permission)
Download a given path to the local file system.
|
static boolean[] |
getArchiveVisibilities(org.apache.hadoop.conf.Configuration conf)
Get the booleans on whether the archives are public or not.
|
static void |
getDelegationTokens(org.apache.hadoop.conf.Configuration job,
org.apache.hadoop.security.Credentials credentials)
For each archive or cache file - get the corresponding delegation token
|
static boolean[] |
getFileVisibilities(org.apache.hadoop.conf.Configuration conf)
Get the booleans on whether the files are public or not.
|
protected TaskDistributedCacheManager |
getTaskDistributedCacheManager(JobID jobId) |
TaskDistributedCacheManager |
newTaskDistributedCacheManager(JobID jobId,
org.apache.hadoop.conf.Configuration taskConf) |
void |
purgeCache()
Clear the entire contents of the cache and delete the backing files.
|
void |
removeTaskDistributedCacheManager(JobID jobId) |
void |
setArchiveSizes(JobID jobId,
long[] sizes)
Set the sizes for any archives, files, or directories in the private
distributed cache.
|
void |
startCleanupThread()
Start the background thread
|
void |
stopCleanupThread()
Stop the background thread
|
static void |
validate(org.apache.hadoop.conf.Configuration conf)
This is part of the framework API.
|
public static final java.lang.String WORK_DIR_FIX
protected TrackerDistributedCacheManager.BaseDirManager baseDirManager
protected TrackerDistributedCacheManager.CleanupThread cleanupThread
public TrackerDistributedCacheManager(org.apache.hadoop.conf.Configuration conf, TaskController controller) throws java.io.IOException
java.io.IOException
public TrackerDistributedCacheManager(org.apache.hadoop.conf.Configuration conf, TaskController taskController, MRAsyncDiskService asyncDiskService) throws java.io.IOException
asyncDiskService
- Provides a set of ThreadPools for async disk
operations.java.io.IOException
public static long downloadCacheObject(org.apache.hadoop.conf.Configuration conf, java.net.URI source, org.apache.hadoop.fs.Path destination, long desiredTimestamp, boolean isArchive, org.apache.hadoop.fs.permission.FsPermission permission) throws java.io.IOException
conf
- the job's configurationsource
- the source to copy fromdestination
- where to copy the file. must be local fsdesiredTimestamp
- the required modification timestamp of the sourceisArchive
- is this an archive that should be expandedpermission
- the desired permissions of the file.java.io.IOException
public static void createAllSymlink(org.apache.hadoop.conf.Configuration conf, java.io.File jobCacheDir, java.io.File workDir) throws java.io.IOException
conf
- the configurationjobCacheDir
- the target directory for creating symlinksworkDir
- the directory in which the symlinks are createdjava.io.IOException
public void purgeCache()
public TaskDistributedCacheManager newTaskDistributedCacheManager(JobID jobId, org.apache.hadoop.conf.Configuration taskConf) throws java.io.IOException
java.io.IOException
public void setArchiveSizes(JobID jobId, long[] sizes) throws java.io.IOException
java.io.IOException
public void removeTaskDistributedCacheManager(JobID jobId)
protected TaskDistributedCacheManager getTaskDistributedCacheManager(JobID jobId)
public static void determineTimestampsAndCacheVisibilities(org.apache.hadoop.conf.Configuration job) throws java.io.IOException
job
- java.io.IOException
public static boolean[] getFileVisibilities(org.apache.hadoop.conf.Configuration conf)
conf
- The configuration which stored the timestampsjava.io.IOException
public static boolean[] getArchiveVisibilities(org.apache.hadoop.conf.Configuration conf)
conf
- The configuration which stored the timestampspublic static void getDelegationTokens(org.apache.hadoop.conf.Configuration job, org.apache.hadoop.security.Credentials credentials) throws java.io.IOException
job
- credentials
- java.io.IOException
public static void validate(org.apache.hadoop.conf.Configuration conf) throws InvalidJobConfException
conf
- a Configuration
to be cheked for duplication
in cached URIsInvalidJobConfException
public void startCleanupThread()
public void stopCleanupThread()
Copyright © 2009 The Apache Software Foundation