Package org.apache.hadoop.mapreduce.v2.app.job.impl

Class JobImpl

java.lang.Object

org.apache.hadoop.mapreduce.v2.app.job.impl.JobImpl

All Implemented Interfaces:

Job, org.apache.hadoop.yarn.event.EventHandler<JobEvent>

public class JobImpl
extends java.lang.Object
implements Job, org.apache.hadoop.yarn.event.EventHandler<JobEvent>

Implementation of Job interface. Maintains the state machines of Job. The read and write calls use ReadWriteLock for concurrency.

Nested Class Summary

Nested Classes

Modifier and Type	Class	Description
static class	JobImpl.InitTransition
static class	JobImpl.StartTransition

Field Summary

Fields

Modifier and Type	Field	Description
org.apache.hadoop.mapred.JobConf	conf
static java.lang.String	JOB_KILLED_DIAG
org.apache.hadoop.fs.Path	remoteJobConfFile
protected static org.apache.hadoop.yarn.state.StateMachineFactory<JobImpl,JobStateInternal,JobEventType,JobEvent>	stateMachineFactory

Constructor Summary

Constructors

Constructor

Description

JobImpl(org.apache.hadoop.mapreduce.v2.api.records.JobId jobId, org.apache.hadoop.yarn.api.records.ApplicationAttemptId applicationAttemptId, org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.yarn.event.EventHandler eventHandler, TaskAttemptListener taskAttemptListener, org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager jobTokenSecretManager, org.apache.hadoop.security.Credentials jobCredentials, org.apache.hadoop.yarn.util.Clock clock, java.util.Map<org.apache.hadoop.mapreduce.v2.api.records.TaskId,org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskInfo> completedTasksFromPreviousRun, MRAppMetrics metrics, org.apache.hadoop.mapreduce.OutputCommitter committer, boolean newApiCommitter, java.lang.String userName, long appSubmitTime, java.util.List<org.apache.hadoop.mapreduce.v2.api.records.AMInfo> amInfos, AppContext appContext, JobStateInternal forcedState, java.lang.String forcedDiagnostic)

Method Summary

Modifier and Type	Method	Description
protected void	addDiagnostic(java.lang.String diag)
protected void	addTask(Task task)
boolean	checkAccess(org.apache.hadoop.security.UserGroupInformation callerUGI, org.apache.hadoop.mapreduce.JobACL jobOperation)
protected JobStateInternal	checkReadyForCommit()
void	constructFinalFullcounters()
static java.lang.String	escapeString(java.lang.String data)
org.apache.hadoop.mapreduce.Counters	getAllCounters()	Get all the counters of this job.
java.util.List<org.apache.hadoop.mapreduce.v2.api.records.AMInfo>	getAMInfos()
int	getCompletedMaps()
int	getCompletedReduces()
org.apache.hadoop.fs.Path	getConfFile()
java.util.List<java.lang.String>	getDiagnostics()
int	getFailedMaps()
int	getFailedReduces()
protected org.apache.hadoop.fs.FileSystem	getFileSystem(org.apache.hadoop.conf.Configuration conf)	Create the default file System for this job.
org.apache.hadoop.mapreduce.v2.api.records.JobId	getID()
JobStateInternal	getInternalState()
java.util.Map<org.apache.hadoop.mapreduce.JobACL,org.apache.hadoop.security.authorize.AccessControlList>	getJobACLs()
int	getKilledMaps()
int	getKilledReduces()
org.apache.hadoop.mapred.TaskCompletionEvent[]	getMapAttemptCompletionEvents(int startIndex, int maxEvents)
float	getMaxAllowedFetchFailuresFraction()
int	getMaxFetchFailuresNotifications()
java.lang.String	getName()
float	getProgress()
java.lang.String	getQueueName()
org.apache.hadoop.mapreduce.v2.api.records.JobReport	getReport()
org.apache.hadoop.mapreduce.v2.api.records.JobState	getState()
protected org.apache.hadoop.yarn.state.StateMachine<JobStateInternal,JobEventType,JobEvent>	getStateMachine()
Task	getTask(org.apache.hadoop.mapreduce.v2.api.records.TaskId taskID)
org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEvent[]	getTaskAttemptCompletionEvents(int fromEventId, int maxEvents)
java.util.Map<org.apache.hadoop.mapreduce.v2.api.records.TaskId,Task>	getTasks()
java.util.Map<org.apache.hadoop.mapreduce.v2.api.records.TaskId,Task>	getTasks(org.apache.hadoop.mapreduce.v2.api.records.TaskType taskType)
int	getTotalMaps()
int	getTotalReduces()
java.lang.String	getUserName()
void	handle(JobEvent event)
static org.apache.hadoop.mapreduce.Counters	incrTaskCounters(org.apache.hadoop.mapreduce.Counters counters, java.util.Collection<Task> tasks)
boolean	isUber()
org.apache.hadoop.conf.Configuration	loadConfFile()
protected void	scheduleTasks(java.util.Set<org.apache.hadoop.mapreduce.v2.api.records.TaskId> taskIDs, boolean recoverTaskOutput)
void	setJobPriority(org.apache.hadoop.yarn.api.records.Priority priority)
void	setQueueName(java.lang.String queueName)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

JOB_KILLED_DIAG

public static final java.lang.String JOB_KILLED_DIAG

See Also:

Constant Field Values

conf

public org.apache.hadoop.mapred.JobConf conf

remoteJobConfFile

public org.apache.hadoop.fs.Path remoteJobConfFile

stateMachineFactory

protected static final org.apache.hadoop.yarn.state.StateMachineFactory<JobImpl,JobStateInternal,JobEventType,JobEvent> stateMachineFactory

Constructor Detail

JobImpl

public JobImpl(org.apache.hadoop.mapreduce.v2.api.records.JobId jobId,
               org.apache.hadoop.yarn.api.records.ApplicationAttemptId applicationAttemptId,
               org.apache.hadoop.conf.Configuration conf,
               org.apache.hadoop.yarn.event.EventHandler eventHandler,
               TaskAttemptListener taskAttemptListener,
               org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager jobTokenSecretManager,
               org.apache.hadoop.security.Credentials jobCredentials,
               org.apache.hadoop.yarn.util.Clock clock,
               java.util.Map<org.apache.hadoop.mapreduce.v2.api.records.TaskId,org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskInfo> completedTasksFromPreviousRun,
               MRAppMetrics metrics,
               org.apache.hadoop.mapreduce.OutputCommitter committer,
               boolean newApiCommitter,
               java.lang.String userName,
               long appSubmitTime,
               java.util.List<org.apache.hadoop.mapreduce.v2.api.records.AMInfo> amInfos,
               AppContext appContext,
               JobStateInternal forcedState,
               java.lang.String forcedDiagnostic)

Method Detail

getStateMachine

protected org.apache.hadoop.yarn.state.StateMachine<JobStateInternal,JobEventType,JobEvent> getStateMachine()

getID

public org.apache.hadoop.mapreduce.v2.api.records.JobId getID()

Specified by:

getID in interface Job

checkAccess

public boolean checkAccess(org.apache.hadoop.security.UserGroupInformation callerUGI,
                           org.apache.hadoop.mapreduce.JobACL jobOperation)

Specified by:

checkAccess in interface Job

getTask

public Task getTask(org.apache.hadoop.mapreduce.v2.api.records.TaskId taskID)

Specified by:

getTask in interface Job

getCompletedMaps

public int getCompletedMaps()

Specified by:

getCompletedMaps in interface Job

getCompletedReduces

public int getCompletedReduces()

Specified by:

getCompletedReduces in interface Job

isUber

public boolean isUber()

Specified by:

isUber in interface Job

getAllCounters

public org.apache.hadoop.mapreduce.Counters getAllCounters()

Description copied from interface: Job

Get all the counters of this job. This includes job-counters aggregated together with the counters of each task. This creates a clone of the Counters, so use this judiciously.

Specified by:

getAllCounters in interface Job

Returns:

job-counters and aggregate task-counters

incrTaskCounters

public static org.apache.hadoop.mapreduce.Counters incrTaskCounters(org.apache.hadoop.mapreduce.Counters counters,
                                                                    java.util.Collection<Task> tasks)

getTaskAttemptCompletionEvents

public org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEvent[] getTaskAttemptCompletionEvents(int fromEventId,
                                                                                                              int maxEvents)

Specified by:

getTaskAttemptCompletionEvents in interface Job

getMapAttemptCompletionEvents

public org.apache.hadoop.mapred.TaskCompletionEvent[] getMapAttemptCompletionEvents(int startIndex,
                                                                                    int maxEvents)

Specified by:

getMapAttemptCompletionEvents in interface Job

getDiagnostics

public java.util.List<java.lang.String> getDiagnostics()

Specified by:

getDiagnostics in interface Job

getReport

public org.apache.hadoop.mapreduce.v2.api.records.JobReport getReport()

Specified by:

getReport in interface Job

getProgress

public float getProgress()

Specified by:

getProgress in interface Job

getTasks

public java.util.Map<org.apache.hadoop.mapreduce.v2.api.records.TaskId,Task> getTasks()

Specified by:

getTasks in interface Job

getTasks

public java.util.Map<org.apache.hadoop.mapreduce.v2.api.records.TaskId,Task> getTasks(org.apache.hadoop.mapreduce.v2.api.records.TaskType taskType)

Specified by:

getTasks in interface Job

getState

public org.apache.hadoop.mapreduce.v2.api.records.JobState getState()

Specified by:

getState in interface Job

scheduleTasks

protected void scheduleTasks(java.util.Set<org.apache.hadoop.mapreduce.v2.api.records.TaskId> taskIDs,
                             boolean recoverTaskOutput)

handle

public void handle(JobEvent event)

Specified by:

handle in interface org.apache.hadoop.yarn.event.EventHandler<JobEvent>

getInternalState

@Private
public JobStateInternal getInternalState()

addTask

protected void addTask(Task task)

getFileSystem

protected org.apache.hadoop.fs.FileSystem getFileSystem(org.apache.hadoop.conf.Configuration conf)
                                                 throws java.io.IOException

Create the default file System for this job.

Parameters:

conf - the conf object

Returns:

the default filesystem for this job

Throws:

java.io.IOException

checkReadyForCommit

protected JobStateInternal checkReadyForCommit()

getUserName

public java.lang.String getUserName()

Specified by:

getUserName in interface Job

getQueueName

public java.lang.String getQueueName()

Specified by:

getQueueName in interface Job

setQueueName

public void setQueueName(java.lang.String queueName)

Specified by:

setQueueName in interface Job

getConfFile

public org.apache.hadoop.fs.Path getConfFile()

Specified by:

getConfFile in interface Job

Returns:

a path to where the config file for this job is located.

getName

public java.lang.String getName()

Specified by:

getName in interface Job

getTotalMaps

public int getTotalMaps()

Specified by:

getTotalMaps in interface Job

getTotalReduces

public int getTotalReduces()

Specified by:

getTotalReduces in interface Job

getJobACLs

public java.util.Map<org.apache.hadoop.mapreduce.JobACL,org.apache.hadoop.security.authorize.AccessControlList> getJobACLs()

Specified by:

getJobACLs in interface Job

Returns:

the ACLs for this job for each type of JobACL given.

getAMInfos

public java.util.List<org.apache.hadoop.mapreduce.v2.api.records.AMInfo> getAMInfos()

Specified by:

getAMInfos in interface Job

Returns:

information for MR AppMasters (previously failed and current)

escapeString

public static java.lang.String escapeString(java.lang.String data)

constructFinalFullcounters

@Private
public void constructFinalFullcounters()

addDiagnostic

protected void addDiagnostic(java.lang.String diag)

loadConfFile

public org.apache.hadoop.conf.Configuration loadConfFile()
                                                  throws java.io.IOException

Specified by:

loadConfFile in interface Job

Returns:

a parsed version of the config files pointed to by Job.getConfFile().

Throws:

java.io.IOException - on any error trying to load the conf file.

getMaxAllowedFetchFailuresFraction

public float getMaxAllowedFetchFailuresFraction()

getMaxFetchFailuresNotifications

public int getMaxFetchFailuresNotifications()

setJobPriority

public void setJobPriority(org.apache.hadoop.yarn.api.records.Priority priority)

Specified by:

setJobPriority in interface Job

getFailedMaps

public int getFailedMaps()

Specified by:

getFailedMaps in interface Job

getFailedReduces

public int getFailedReduces()

Specified by:

getFailedReduces in interface Job

getKilledMaps

public int getKilledMaps()

Specified by:

getKilledMaps in interface Job

getKilledReduces

public int getKilledReduces()

Specified by:

getKilledReduces in interface Job