public class TaskTracker extends java.lang.Object implements TaskUmbilicalProtocol, java.lang.Runnable, TaskTrackerMXBean
Modifier and Type | Class and Description |
---|---|
static class |
TaskTracker.DefaultShuffleProvider |
static class |
TaskTracker.MapOutputServlet
This class is used in TaskTracker's Jetty to serve the map outputs
to other nodes.
|
Modifier and Type | Field and Description |
---|---|
static org.apache.commons.logging.Log |
ClientTraceLog |
static long |
COUNTER_UPDATE_INTERVAL |
static long |
DEFAULT_DISK_HEALTH_CHECK_INTERVAL
How often TaskTracker needs to check the health of its disks, if not
configured using mapred.disk.healthChecker.interval
|
static int |
FILE_NOT_FOUND |
static java.lang.String |
FOR_REDUCE_TASK
The reduce task number for which this map output is being transferred
|
static java.lang.String |
FROM_MAP_TASK
The map task from which the map output data is being transferred
|
static int |
HEARTBEAT_INTERVAL_MIN_DEFAULT |
static org.apache.commons.logging.Log |
LOG |
static java.lang.String |
MAP_OUTPUT_LENGTH
The custom http header used for the map output length.
|
static java.lang.String |
MR_CLIENTTRACE_FORMAT |
static java.lang.String |
RAW_MAP_OUTPUT_LENGTH
The custom http header used for the "raw" map output length.
|
static java.lang.String |
SHUFFLE_PROVIDER_PLUGIN_CLASSES |
static java.lang.String |
SUBDIR |
static int |
SUCCESS |
static java.lang.String |
TT_KEYTAB_FILE |
static java.lang.String |
TT_LOG_TMP_DIR |
static java.lang.String |
TT_RESOURCE_CALCULATOR_PLUGIN |
static java.lang.String |
TT_USER_NAME |
static java.lang.String |
WORKDIR |
versionID
Constructor and Description |
---|
TaskTracker(JobConf conf)
Start with the local machine name, and the default JobTracker
|
Modifier and Type | Method and Description |
---|---|
boolean |
canCommit(TaskAttemptID taskid,
org.apache.hadoop.mapred.JvmContext jvmContext)
Child checking whether it can commit
|
void |
cleanupStorage()
Deprecated.
|
void |
close()
Close down the TaskTracker and all its components.
|
void |
commitPending(TaskAttemptID taskid,
TaskStatus taskStatus,
org.apache.hadoop.mapred.JvmContext jvmContext)
Task is reporting that it is in commit_pending
and it is waiting for the commit Response
|
void |
done(TaskAttemptID taskid,
org.apache.hadoop.mapred.JvmContext jvmContext)
The task is done.
|
void |
fatalError(TaskAttemptID taskId,
java.lang.String msg,
org.apache.hadoop.mapred.JvmContext jvmContext)
A child task had a fatal error.
|
void |
fsError(TaskAttemptID taskId,
java.lang.String message,
org.apache.hadoop.mapred.JvmContext jvmContext)
A child task had a local filesystem error.
|
java.lang.String |
getConfigVersion() |
java.lang.String |
getHostname() |
int |
getHttpPort() |
static java.lang.Class<? extends org.apache.hadoop.mapred.TaskTrackerInstrumentation> |
getInstrumentationClass(org.apache.hadoop.conf.Configuration conf) |
static java.lang.String |
getIntermediateOutputDir(java.lang.String user,
java.lang.String jobid,
java.lang.String taskid) |
static java.lang.String |
getJobCacheSubdir(java.lang.String user) |
org.apache.hadoop.mapred.InterTrackerProtocol |
getJobClient()
The connection to the JobTracker, used by the TaskRunner
for locating remote files.
|
JobConf |
getJobConf()
Get the default job conf for this tracker.
|
JobConf |
getJobConf(JobID jobId)
Get the specific job conf for a running job.
|
static java.lang.String |
getJobJarFile(java.lang.String user,
java.lang.String jobid) |
java.lang.String |
getJobTrackerUrl() |
org.apache.hadoop.mapred.JvmManager |
getJvmManagerInstance() |
static java.lang.String |
getLocalJobDir(java.lang.String user,
java.lang.String jobid) |
static java.lang.String |
getLocalTaskDir(java.lang.String user,
java.lang.String jobid,
java.lang.String taskid) |
static java.lang.String |
getLocalTaskDir(java.lang.String user,
java.lang.String jobid,
java.lang.String taskid,
boolean isCleanupAttempt) |
MapTaskCompletionEventsUpdate |
getMapCompletionEvents(JobID jobId,
int fromEventId,
int maxLocs,
TaskAttemptID id,
org.apache.hadoop.mapred.JvmContext jvmContext)
Called by a reduce task to get the map output locations for finished maps.
|
static java.lang.String |
getPrivateDistributedCacheDir(java.lang.String user) |
org.apache.hadoop.ipc.ProtocolSignature |
getProtocolSignature(java.lang.String protocol,
long clientVersion,
int clientMethodsHash) |
long |
getProtocolVersion(java.lang.String protocol,
long clientVersion) |
static java.lang.String |
getPublicDistributedCacheDir() |
int |
getRpcPort() |
JvmTask |
getTask(org.apache.hadoop.mapred.JvmContext context)
Called upon startup by the child process, to fetch Task data.
|
TaskController |
getTaskController() |
org.apache.hadoop.mapred.TaskMemoryManagerThread |
getTaskMemoryManager() |
java.lang.String |
getTasksInfoJson() |
org.apache.hadoop.mapred.TaskTrackerInstrumentation |
getTaskTrackerInstrumentation() |
java.net.InetSocketAddress |
getTaskTrackerReportAddress()
Return the port at which the tasktracker bound to
|
static java.lang.String |
getUserDir(java.lang.String user) |
java.lang.String |
getVersion() |
boolean |
isHealthy() |
boolean |
isIdle()
Is this task tracker idle?
|
boolean |
isTaskMemoryManagerEnabled()
Is the TaskMemoryManager Enabled on this system?
|
static void |
main(java.lang.String[] argv)
Start the TaskTracker, point toward the indicated JobTracker
|
void |
mapOutputLost(TaskAttemptID taskid,
java.lang.String errorMsg)
A completed map task's output has been lost.
|
boolean |
ping(TaskAttemptID taskid,
org.apache.hadoop.mapred.JvmContext jvmContext)
Child checking to see if we're alive.
|
void |
reportDiagnosticInfo(TaskAttemptID taskid,
java.lang.String info,
org.apache.hadoop.mapred.JvmContext jvmContext)
Called when the task dies before completion, and we want to report back
diagnostic info
|
void |
reportNextRecordRange(TaskAttemptID taskid,
org.apache.hadoop.mapred.SortedRanges.Range range,
org.apache.hadoop.mapred.JvmContext jvmContext)
Report the record range which is going to process next by the Task.
|
void |
run()
The server retry loop.
|
static void |
setInstrumentationClass(org.apache.hadoop.conf.Configuration conf,
java.lang.Class<? extends org.apache.hadoop.mapred.TaskTrackerInstrumentation> t) |
void |
shuffleError(TaskAttemptID taskId,
java.lang.String message,
org.apache.hadoop.mapred.JvmContext jvmContext)
A reduce-task failed to shuffle the map-outputs.
|
void |
shutdown() |
boolean |
statusUpdate(TaskAttemptID taskid,
TaskStatus taskStatus,
org.apache.hadoop.mapred.JvmContext jvmContext)
Called periodically to report Task progress, from 0.0 to 1.0.
|
void |
updatePrivateDistributedCacheSizes(JobID jobId,
long[] sizes)
The job initializer needs to report the sizes of the archive
objects and directories in the private distributed cache.
|
public static final java.lang.String SHUFFLE_PROVIDER_PLUGIN_CLASSES
public static final org.apache.commons.logging.Log LOG
public static final java.lang.String MR_CLIENTTRACE_FORMAT
public static final org.apache.commons.logging.Log ClientTraceLog
public static final java.lang.String SUBDIR
public static final java.lang.String TT_LOG_TMP_DIR
public static final java.lang.String TT_RESOURCE_CALCULATOR_PLUGIN
public static final java.lang.String TT_USER_NAME
public static final java.lang.String TT_KEYTAB_FILE
public static final int HEARTBEAT_INTERVAL_MIN_DEFAULT
public static final long COUNTER_UPDATE_INTERVAL
public static final long DEFAULT_DISK_HEALTH_CHECK_INTERVAL
public static final int SUCCESS
public static final int FILE_NOT_FOUND
public static final java.lang.String MAP_OUTPUT_LENGTH
public static final java.lang.String RAW_MAP_OUTPUT_LENGTH
public static final java.lang.String FROM_MAP_TASK
public static final java.lang.String FOR_REDUCE_TASK
public static final java.lang.String WORKDIR
public TaskTracker(JobConf conf) throws java.io.IOException, java.lang.InterruptedException
java.io.IOException
java.lang.InterruptedException
public org.apache.hadoop.mapred.TaskTrackerInstrumentation getTaskTrackerInstrumentation()
public TaskController getTaskController()
public static java.lang.String getUserDir(java.lang.String user)
public static java.lang.String getPrivateDistributedCacheDir(java.lang.String user)
public static java.lang.String getPublicDistributedCacheDir()
public static java.lang.String getJobCacheSubdir(java.lang.String user)
public static java.lang.String getLocalJobDir(java.lang.String user, java.lang.String jobid)
public static java.lang.String getJobJarFile(java.lang.String user, java.lang.String jobid)
public static java.lang.String getIntermediateOutputDir(java.lang.String user, java.lang.String jobid, java.lang.String taskid)
public static java.lang.String getLocalTaskDir(java.lang.String user, java.lang.String jobid, java.lang.String taskid)
public static java.lang.String getLocalTaskDir(java.lang.String user, java.lang.String jobid, java.lang.String taskid, boolean isCleanupAttempt)
public long getProtocolVersion(java.lang.String protocol, long clientVersion) throws java.io.IOException
getProtocolVersion
in interface org.apache.hadoop.ipc.VersionedProtocol
java.io.IOException
public org.apache.hadoop.ipc.ProtocolSignature getProtocolSignature(java.lang.String protocol, long clientVersion, int clientMethodsHash) throws java.io.IOException
getProtocolSignature
in interface org.apache.hadoop.ipc.VersionedProtocol
java.io.IOException
public static java.lang.Class<? extends org.apache.hadoop.mapred.TaskTrackerInstrumentation> getInstrumentationClass(org.apache.hadoop.conf.Configuration conf)
public static void setInstrumentationClass(org.apache.hadoop.conf.Configuration conf, java.lang.Class<? extends org.apache.hadoop.mapred.TaskTrackerInstrumentation> t)
@Deprecated public void cleanupStorage() throws java.io.IOException
java.io.IOException
org.apache.hadoop.mapreduce.util.MRAsyncDiskService#cleanupDirsInAllVolumes()
public void shutdown() throws java.io.IOException, java.lang.InterruptedException
java.io.IOException
java.lang.InterruptedException
public void close() throws java.io.IOException, java.lang.InterruptedException
java.lang.InterruptedException
java.io.IOException
public org.apache.hadoop.mapred.InterTrackerProtocol getJobClient()
public java.net.InetSocketAddress getTaskTrackerReportAddress()
public org.apache.hadoop.mapred.JvmManager getJvmManagerInstance()
public void run()
run
in interface java.lang.Runnable
public JvmTask getTask(org.apache.hadoop.mapred.JvmContext context) throws java.io.IOException
getTask
in interface TaskUmbilicalProtocol
context
- the JvmContext of the JVM w.r.t the TaskTracker that
launched itjava.io.IOException
public boolean statusUpdate(TaskAttemptID taskid, TaskStatus taskStatus, org.apache.hadoop.mapred.JvmContext jvmContext) throws java.io.IOException
statusUpdate
in interface TaskUmbilicalProtocol
taskid
- task-id of the childtaskStatus
- status of the childjvmContext
- context the jvmContext running the task.java.io.IOException
public void reportDiagnosticInfo(TaskAttemptID taskid, java.lang.String info, org.apache.hadoop.mapred.JvmContext jvmContext) throws java.io.IOException
reportDiagnosticInfo
in interface TaskUmbilicalProtocol
taskid
- the id of the task involvedinfo
- the text to reportjvmContext
- context the jvmContext running the task.java.io.IOException
public void reportNextRecordRange(TaskAttemptID taskid, org.apache.hadoop.mapred.SortedRanges.Range range, org.apache.hadoop.mapred.JvmContext jvmContext) throws java.io.IOException
TaskUmbilicalProtocol
reportNextRecordRange
in interface TaskUmbilicalProtocol
taskid
- the id of the task involvedrange
- the range of record sequence nosjvmContext
- context the jvmContext running the task.java.io.IOException
public boolean ping(TaskAttemptID taskid, org.apache.hadoop.mapred.JvmContext jvmContext) throws java.io.IOException
ping
in interface TaskUmbilicalProtocol
taskid
- the id of the task involvedjvmContext
- context the jvmContext running the task.java.io.IOException
public void commitPending(TaskAttemptID taskid, TaskStatus taskStatus, org.apache.hadoop.mapred.JvmContext jvmContext) throws java.io.IOException
commitPending
in interface TaskUmbilicalProtocol
taskid
- task's idtaskStatus
- status of the childjvmContext
- context the jvmContext running the task.java.io.IOException
public boolean canCommit(TaskAttemptID taskid, org.apache.hadoop.mapred.JvmContext jvmContext) throws java.io.IOException
canCommit
in interface TaskUmbilicalProtocol
jvmContext
- context the jvmContext running the task.java.io.IOException
public void done(TaskAttemptID taskid, org.apache.hadoop.mapred.JvmContext jvmContext) throws java.io.IOException
done
in interface TaskUmbilicalProtocol
taskid
- task's idjvmContext
- context the jvmContext running the task.java.io.IOException
public void shuffleError(TaskAttemptID taskId, java.lang.String message, org.apache.hadoop.mapred.JvmContext jvmContext) throws java.io.IOException
shuffleError
in interface TaskUmbilicalProtocol
java.io.IOException
public void fsError(TaskAttemptID taskId, java.lang.String message, org.apache.hadoop.mapred.JvmContext jvmContext) throws java.io.IOException
fsError
in interface TaskUmbilicalProtocol
java.io.IOException
public void fatalError(TaskAttemptID taskId, java.lang.String msg, org.apache.hadoop.mapred.JvmContext jvmContext) throws java.io.IOException
fatalError
in interface TaskUmbilicalProtocol
java.io.IOException
public MapTaskCompletionEventsUpdate getMapCompletionEvents(JobID jobId, int fromEventId, int maxLocs, TaskAttemptID id, org.apache.hadoop.mapred.JvmContext jvmContext) throws java.io.IOException
TaskUmbilicalProtocol
getMapCompletionEvents
in interface TaskUmbilicalProtocol
jobId
- the reducer job idfromEventId
- the index starting from which the locations should be
fetchedmaxLocs
- the max number of locations to fetchid
- The attempt id of the task that is trying to communicateMapTaskCompletionEventsUpdate
java.io.IOException
public void mapOutputLost(TaskAttemptID taskid, java.lang.String errorMsg) throws java.io.IOException
java.io.IOException
public JobConf getJobConf(JobID jobId) throws java.io.IOException
java.io.IOException
public JobConf getJobConf()
public boolean isIdle()
public static void main(java.lang.String[] argv) throws java.lang.Exception
java.lang.Exception
public boolean isTaskMemoryManagerEnabled()
public org.apache.hadoop.mapred.TaskMemoryManagerThread getTaskMemoryManager()
public java.lang.String getHostname()
getHostname
in interface TaskTrackerMXBean
public java.lang.String getVersion()
getVersion
in interface TaskTrackerMXBean
public java.lang.String getConfigVersion()
getConfigVersion
in interface TaskTrackerMXBean
public java.lang.String getJobTrackerUrl()
getJobTrackerUrl
in interface TaskTrackerMXBean
public int getRpcPort()
getRpcPort
in interface TaskTrackerMXBean
public int getHttpPort()
getHttpPort
in interface TaskTrackerMXBean
public boolean isHealthy()
isHealthy
in interface TaskTrackerMXBean
public java.lang.String getTasksInfoJson()
getTasksInfoJson
in interface TaskTrackerMXBean
public void updatePrivateDistributedCacheSizes(JobID jobId, long[] sizes) throws java.io.IOException
TaskUmbilicalProtocol
updatePrivateDistributedCacheSizes
in interface TaskUmbilicalProtocol
jobId
- the job to updatesizes
- the array of sizes that were computedjava.io.IOException
Copyright © 2009 The Apache Software Foundation