public class JobInProgress
extends java.lang.Object
Modifier and Type | Class and Description |
---|---|
static class |
JobInProgress.Counter |
Modifier and Type | Field and Description |
---|---|
protected org.apache.hadoop.security.Credentials |
tokenStorage |
Modifier | Constructor and Description |
---|---|
protected |
JobInProgress(JobID jobid,
JobConf conf,
JobTracker tracker)
Create an almost empty JobInProgress, which can be used only for tests
|
Modifier and Type | Method and Description |
---|---|
void |
cleanUpMetrics()
Called when the job is complete
|
boolean |
completedTask(TaskInProgress tip,
TaskStatus status)
A taskid assigned to this JobInProgress has reported in successfully.
|
int |
desiredMaps() |
int |
desiredReduces() |
int |
desiredTasks()
Return total number of map and reduce tasks desired by the job.
|
void |
failedTask(TaskInProgress tip,
TaskAttemptID taskid,
java.lang.String reason,
TaskStatus.Phase phase,
TaskStatus.State state,
java.lang.String trackerName)
Fail a task with a given reason, but without a status object.
|
TaskStatus |
findFinishedMap(int mapId)
Find the details of someplace where a map has finished
|
protected TaskInProgress |
findSpeculativeTask(java.util.Collection<TaskInProgress> list,
TaskTrackerStatus ttStatus,
double avgProgress,
long currentTime,
boolean shouldRemove)
Find a speculative task
|
int |
finishedMaps() |
int |
finishedReduces() |
Counters |
getCounters()
Returns the total job counters, by adding together the job,
the map and the reduce counters.
|
long |
getFinishTime() |
Counters |
getJobCounters()
Returns the job-level counters.
|
JobID |
getJobID() |
java.lang.String |
getJobSubmitHostAddress() |
java.lang.String |
getJobSubmitHostName() |
long |
getLaunchTime() |
Counters |
getMapCounters()
Returns map phase counters by summing over all map tasks in progress.
|
int |
getNumReservedTaskTrackersForMaps() |
int |
getNumReservedTaskTrackersForReduces() |
long |
getNumSchedulingOpportunities() |
int |
getNumSlotsPerTask(TaskType taskType) |
JobPriority |
getPriority() |
JobProfile |
getProfile() |
Counters |
getReduceCounters()
Returns map phase counters by summing over all map tasks in progress.
|
java.lang.Object |
getSchedulingInfo() |
long |
getStartTime() |
JobStatus |
getStatus() |
TaskCompletionEvent[] |
getTaskCompletionEvents(int fromEventId,
int maxEvents) |
TaskInProgress |
getTaskInProgress(TaskID tipid)
Return the TaskInProgress that matches the tipid.
|
java.lang.String |
getUser()
Get the user for the job
|
boolean |
hasSpeculativeMaps() |
boolean |
hasSpeculativeReduces() |
boolean |
inited()
Check if the job has been initialized.
|
void |
initTasks()
Construct the splits, etc.
|
void |
kill()
Kill the job and all its component tasks.
|
Task |
obtainJobCleanupTask(TaskTrackerStatus tts,
int clusterSize,
int numUniqueHosts,
boolean isMapSlot)
Return a CleanupTask, if appropriate, to run on the given tasktracker
|
Task |
obtainJobSetupTask(TaskTrackerStatus tts,
int clusterSize,
int numUniqueHosts,
boolean isMapSlot)
Return a SetupTask, if appropriate, to run on the given tasktracker
|
Task |
obtainNewLocalMapTask(TaskTrackerStatus tts,
int clusterSize,
int numUniqueHosts) |
Task |
obtainNewMapTask(TaskTrackerStatus tts,
int clusterSize,
int numUniqueHosts)
Return a MapTask, if appropriate, to run on the given tasktracker
|
Task |
obtainNewMapTask(TaskTrackerStatus tts,
int clusterSize,
int numUniqueHosts,
int maxCacheLevel)
Return a MapTask, if appropriate, to run on the given tasktracker
|
Task |
obtainNewNonLocalMapTask(TaskTrackerStatus tts,
int clusterSize,
int numUniqueHosts) |
Task |
obtainNewReduceTask(TaskTrackerStatus tts,
int clusterSize,
int numUniqueHosts)
Return a ReduceTask, if appropriate, to run on the given tasktracker.
|
Task |
obtainTaskCleanupTask(TaskTrackerStatus tts,
boolean isMapSlot) |
void |
overrideSchedulingOpportunities() |
int |
pendingMaps() |
int |
pendingReduces() |
java.util.Vector<TaskInProgress> |
reportCleanupTIPs(boolean shouldBeComplete)
Return a vector of cleanup TaskInProgress objects
|
java.util.Vector<TaskInProgress> |
reportSetupTIPs(boolean shouldBeComplete)
Return a vector of setup TaskInProgress objects
|
java.util.Vector<TaskInProgress> |
reportTasksInProgress(boolean shouldBeMap,
boolean shouldBeComplete)
Return a vector of completed TaskInProgress objects
|
void |
reserveTaskTracker(TaskTracker taskTracker,
TaskType type,
int numSlots) |
void |
resetSchedulingOpportunities() |
int |
runningMaps() |
int |
runningReduces() |
protected void |
scheduleMap(TaskInProgress tip)
Adds a map tip to the list of running maps.
|
boolean |
scheduleOffSwitch(int numTaskTrackers)
Check if we can schedule an off-switch task for this job.
|
protected void |
scheduleReduce(TaskInProgress tip)
Adds a reduce tip to the list of running reduces
|
boolean |
scheduleReduces() |
void |
schedulingOpportunity() |
void |
setPriority(JobPriority priority) |
void |
setSchedulingInfo(java.lang.Object schedulingInfo) |
void |
unreserveTaskTracker(TaskTracker taskTracker,
TaskType type) |
void |
updateMetrics()
Called periodically by JobTrackerMetrics to update the metrics for
this job.
|
void |
updateTaskStatus(TaskInProgress tip,
TaskStatus status)
Assuming
JobTracker is locked on entry. |
protected JobInProgress(JobID jobid, JobConf conf, JobTracker tracker) throws java.io.IOException
java.io.IOException
public boolean hasSpeculativeMaps()
public boolean hasSpeculativeReduces()
public void updateMetrics()
public void cleanUpMetrics()
public boolean inited()
true
if the job has been initialized,
false
otherwisepublic java.lang.String getUser()
public void initTasks() throws java.io.IOException, org.apache.hadoop.mapred.JobInProgress.KillInterruptedException
java.io.IOException
org.apache.hadoop.mapred.JobInProgress.KillInterruptedException
public JobProfile getProfile()
public JobStatus getStatus()
public long getLaunchTime()
public long getStartTime()
public long getFinishTime()
public int desiredMaps()
public int finishedMaps()
public int desiredReduces()
public int runningMaps()
public int runningReduces()
public int finishedReduces()
public int pendingMaps()
public int pendingReduces()
public int desiredTasks()
public int getNumSlotsPerTask(TaskType taskType)
public JobPriority getPriority()
public void setPriority(JobPriority priority)
public java.util.Vector<TaskInProgress> reportTasksInProgress(boolean shouldBeMap, boolean shouldBeComplete)
public java.util.Vector<TaskInProgress> reportCleanupTIPs(boolean shouldBeComplete)
public java.util.Vector<TaskInProgress> reportSetupTIPs(boolean shouldBeComplete)
public void updateTaskStatus(TaskInProgress tip, TaskStatus status)
JobTracker
is locked on entry.public Counters getJobCounters()
public Counters getMapCounters()
public Counters getReduceCounters()
public Counters getCounters()
public Task obtainNewMapTask(TaskTrackerStatus tts, int clusterSize, int numUniqueHosts, int maxCacheLevel) throws java.io.IOException
java.io.IOException
public Task obtainNewMapTask(TaskTrackerStatus tts, int clusterSize, int numUniqueHosts) throws java.io.IOException
java.io.IOException
public Task obtainTaskCleanupTask(TaskTrackerStatus tts, boolean isMapSlot) throws java.io.IOException
java.io.IOException
public Task obtainNewLocalMapTask(TaskTrackerStatus tts, int clusterSize, int numUniqueHosts) throws java.io.IOException
java.io.IOException
public Task obtainNewNonLocalMapTask(TaskTrackerStatus tts, int clusterSize, int numUniqueHosts) throws java.io.IOException
java.io.IOException
public void schedulingOpportunity()
public void resetSchedulingOpportunities()
public long getNumSchedulingOpportunities()
public void overrideSchedulingOpportunities()
public boolean scheduleOffSwitch(int numTaskTrackers)
numTaskTrackers
- number of tasktrackerstrue
if we can schedule off-switch,
false
otherwise
We check the number of missed opportunities for the job.
If it has 'waited' long enough we go ahead and schedule.public Task obtainJobCleanupTask(TaskTrackerStatus tts, int clusterSize, int numUniqueHosts, boolean isMapSlot) throws java.io.IOException
java.io.IOException
public Task obtainJobSetupTask(TaskTrackerStatus tts, int clusterSize, int numUniqueHosts, boolean isMapSlot) throws java.io.IOException
java.io.IOException
public boolean scheduleReduces()
public Task obtainNewReduceTask(TaskTrackerStatus tts, int clusterSize, int numUniqueHosts) throws java.io.IOException
java.io.IOException
public void reserveTaskTracker(TaskTracker taskTracker, TaskType type, int numSlots)
public void unreserveTaskTracker(TaskTracker taskTracker, TaskType type)
public int getNumReservedTaskTrackersForMaps()
public int getNumReservedTaskTrackersForReduces()
protected void scheduleMap(TaskInProgress tip)
tip
- the tip that needs to be scheduled as runningprotected void scheduleReduce(TaskInProgress tip)
tip
- the tip that needs to be scheduled as runningprotected TaskInProgress findSpeculativeTask(java.util.Collection<TaskInProgress> list, TaskTrackerStatus ttStatus, double avgProgress, long currentTime, boolean shouldRemove)
list
- a list of tipsttStatus
- status of the tracker that has requested a tipavgProgress
- the average progress for speculationcurrentTime
- current time in millisecondsshouldRemove
- whether to remove the tipspublic boolean completedTask(TaskInProgress tip, TaskStatus status)
public void kill()
public void failedTask(TaskInProgress tip, TaskAttemptID taskid, java.lang.String reason, TaskStatus.Phase phase, TaskStatus.State state, java.lang.String trackerName)
JobTracker
is locked on entry.tip
- The task's tiptaskid
- The task idreason
- The reason that the task failedtrackerName
- The task tracker the task failed onpublic TaskInProgress getTaskInProgress(TaskID tipid)
public TaskStatus findFinishedMap(int mapId)
mapId
- the id of the mappublic TaskCompletionEvent[] getTaskCompletionEvents(int fromEventId, int maxEvents)
public JobID getJobID()
public java.lang.String getJobSubmitHostName()
public java.lang.String getJobSubmitHostAddress()
public java.lang.Object getSchedulingInfo()
public void setSchedulingInfo(java.lang.Object schedulingInfo)
Copyright © 2009 The Apache Software Foundation