org.apache.hadoop.mapreduce.v2.app.job.impl
Class JobImpl

java.lang.Object
  org.apache.hadoop.mapreduce.v2.app.job.impl.JobImpl

All Implemented Interfaces:

Job, org.apache.hadoop.yarn.event.EventHandler<JobEvent>

public class JobImpl
extends Object
implements Job, org.apache.hadoop.yarn.event.EventHandler<JobEvent>

Implementation of Job interface. Maintains the state machines of Job. The read and write calls use ReadWriteLock for concurrency.

Nested Class Summary

static class	JobImpl.InitTransition
static class	JobImpl.StartTransition

Field Summary

org.apache.hadoop.mapred.JobConf	conf
org.apache.hadoop.fs.Path	remoteJobConfFile
protected static org.apache.hadoop.yarn.state.StateMachineFactory<JobImpl,JobStateInternal,JobEventType,JobEvent>	stateMachineFactory

Constructor Summary

JobImpl(org.apache.hadoop.mapreduce.v2.api.records.JobId jobId, org.apache.hadoop.yarn.api.records.ApplicationAttemptId applicationAttemptId, org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.yarn.event.EventHandler eventHandler, TaskAttemptListener taskAttemptListener, org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager jobTokenSecretManager, org.apache.hadoop.security.Credentials jobCredentials, org.apache.hadoop.yarn.util.Clock clock, Map<org.apache.hadoop.mapreduce.v2.api.records.TaskId,org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskInfo> completedTasksFromPreviousRun, MRAppMetrics metrics, org.apache.hadoop.mapreduce.OutputCommitter committer, boolean newApiCommitter, String userName, long appSubmitTime, List<org.apache.hadoop.mapreduce.v2.api.records.AMInfo> amInfos, AppContext appContext, JobStateInternal forcedState, String forcedDiagnostic)

Method Summary

protected void	addDiagnostic(String diag)
protected void	addTask(Task task)
boolean	checkAccess(org.apache.hadoop.security.UserGroupInformation callerUGI, org.apache.hadoop.mapreduce.JobACL jobOperation)
protected JobStateInternal	checkReadyForCommit()
void	constructFinalFullcounters()
static String	escapeString(String data)
org.apache.hadoop.mapreduce.Counters	getAllCounters() Get all the counters of this job.
List<org.apache.hadoop.mapreduce.v2.api.records.AMInfo>	getAMInfos()
int	getCompletedMaps()
int	getCompletedReduces()
org.apache.hadoop.fs.Path	getConfFile()
List<String>	getDiagnostics()
protected org.apache.hadoop.fs.FileSystem	getFileSystem(org.apache.hadoop.conf.Configuration conf) Create the default file System for this job.
org.apache.hadoop.mapreduce.v2.api.records.JobId	getID()
JobStateInternal	getInternalState()
Map<org.apache.hadoop.mapreduce.JobACL,org.apache.hadoop.security.authorize.AccessControlList>	getJobACLs()
org.apache.hadoop.mapred.TaskCompletionEvent[]	getMapAttemptCompletionEvents(int startIndex, int maxEvents)
String	getName()
float	getProgress()
String	getQueueName()
org.apache.hadoop.mapreduce.v2.api.records.JobReport	getReport()
org.apache.hadoop.mapreduce.v2.api.records.JobState	getState()
protected org.apache.hadoop.yarn.state.StateMachine<JobStateInternal,JobEventType,JobEvent>	getStateMachine()
Task	getTask(org.apache.hadoop.mapreduce.v2.api.records.TaskId taskID)
org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEvent[]	getTaskAttemptCompletionEvents(int fromEventId, int maxEvents)
Map<org.apache.hadoop.mapreduce.v2.api.records.TaskId,Task>	getTasks()
Map<org.apache.hadoop.mapreduce.v2.api.records.TaskId,Task>	getTasks(org.apache.hadoop.mapreduce.v2.api.records.TaskType taskType)
int	getTotalMaps()
int	getTotalReduces()
String	getUserName()
void	handle(JobEvent event)
static org.apache.hadoop.mapreduce.Counters	incrTaskCounters(org.apache.hadoop.mapreduce.Counters counters, Collection<Task> tasks)
boolean	isUber()
org.apache.hadoop.conf.Configuration	loadConfFile()
protected void	scheduleTasks(Set<org.apache.hadoop.mapreduce.v2.api.records.TaskId> taskIDs, boolean recoverTaskOutput)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

conf

public org.apache.hadoop.mapred.JobConf conf

remoteJobConfFile

public org.apache.hadoop.fs.Path remoteJobConfFile

stateMachineFactory

protected static final org.apache.hadoop.yarn.state.StateMachineFactory<JobImpl,JobStateInternal,JobEventType,JobEvent> stateMachineFactory

Constructor Detail

JobImpl

public JobImpl(org.apache.hadoop.mapreduce.v2.api.records.JobId jobId,
               org.apache.hadoop.yarn.api.records.ApplicationAttemptId applicationAttemptId,
               org.apache.hadoop.conf.Configuration conf,
               org.apache.hadoop.yarn.event.EventHandler eventHandler,
               TaskAttemptListener taskAttemptListener,
               org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager jobTokenSecretManager,
               org.apache.hadoop.security.Credentials jobCredentials,
               org.apache.hadoop.yarn.util.Clock clock,
               Map<org.apache.hadoop.mapreduce.v2.api.records.TaskId,org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskInfo> completedTasksFromPreviousRun,
               MRAppMetrics metrics,
               org.apache.hadoop.mapreduce.OutputCommitter committer,
               boolean newApiCommitter,
               String userName,
               long appSubmitTime,
               List<org.apache.hadoop.mapreduce.v2.api.records.AMInfo> amInfos,
               AppContext appContext,
               JobStateInternal forcedState,
               String forcedDiagnostic)

Method Detail

getStateMachine

protected org.apache.hadoop.yarn.state.StateMachine<JobStateInternal,JobEventType,JobEvent> getStateMachine()

getID

public org.apache.hadoop.mapreduce.v2.api.records.JobId getID()

Specified by:

getID in interface Job

checkAccess

public boolean checkAccess(org.apache.hadoop.security.UserGroupInformation callerUGI,
                           org.apache.hadoop.mapreduce.JobACL jobOperation)

Specified by:

checkAccess in interface Job

getTask

public Task getTask(org.apache.hadoop.mapreduce.v2.api.records.TaskId taskID)

Specified by:

getTask in interface Job

getCompletedMaps

public int getCompletedMaps()

Specified by:

getCompletedMaps in interface Job

getCompletedReduces

public int getCompletedReduces()

Specified by:

getCompletedReduces in interface Job

isUber

public boolean isUber()

Specified by:

isUber in interface Job

getAllCounters

public org.apache.hadoop.mapreduce.Counters getAllCounters()

Description copied from interface: Job

Get all the counters of this job. This includes job-counters aggregated together with the counters of each task. This creates a clone of the Counters, so use this judiciously.

Specified by:

getAllCounters in interface Job

Returns:

job-counters and aggregate task-counters

incrTaskCounters

public static org.apache.hadoop.mapreduce.Counters incrTaskCounters(org.apache.hadoop.mapreduce.Counters counters,
                                                                    Collection<Task> tasks)

getTaskAttemptCompletionEvents

public org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEvent[] getTaskAttemptCompletionEvents(int fromEventId,
                                                                                                              int maxEvents)

Specified by:

getTaskAttemptCompletionEvents in interface Job

getMapAttemptCompletionEvents

public org.apache.hadoop.mapred.TaskCompletionEvent[] getMapAttemptCompletionEvents(int startIndex,
                                                                                    int maxEvents)

Specified by:

getMapAttemptCompletionEvents in interface Job

getDiagnostics

public List<String> getDiagnostics()

Specified by:

getDiagnostics in interface Job

getReport

public org.apache.hadoop.mapreduce.v2.api.records.JobReport getReport()

Specified by:

getReport in interface Job

getProgress

public float getProgress()

Specified by:

getProgress in interface Job

getTasks

public Map<org.apache.hadoop.mapreduce.v2.api.records.TaskId,Task> getTasks()

Specified by:

getTasks in interface Job

getTasks

public Map<org.apache.hadoop.mapreduce.v2.api.records.TaskId,Task> getTasks(org.apache.hadoop.mapreduce.v2.api.records.TaskType taskType)

Specified by:

getTasks in interface Job

getState

public org.apache.hadoop.mapreduce.v2.api.records.JobState getState()

Specified by:

getState in interface Job

scheduleTasks

protected void scheduleTasks(Set<org.apache.hadoop.mapreduce.v2.api.records.TaskId> taskIDs,
                             boolean recoverTaskOutput)

handle

public void handle(JobEvent event)

Specified by:

handle in interface org.apache.hadoop.yarn.event.EventHandler<JobEvent>

getInternalState

@InterfaceAudience.Private
public JobStateInternal getInternalState()

addTask

protected void addTask(Task task)

getFileSystem

protected org.apache.hadoop.fs.FileSystem getFileSystem(org.apache.hadoop.conf.Configuration conf)
                                                 throws IOException

Create the default file System for this job.

Parameters:

conf - the conf object

Returns:

the default filesystem for this job

Throws:

IOException

checkReadyForCommit

protected JobStateInternal checkReadyForCommit()

getUserName

public String getUserName()

Specified by:

getUserName in interface Job

getQueueName

public String getQueueName()

Specified by:

getQueueName in interface Job

getConfFile

public org.apache.hadoop.fs.Path getConfFile()

Specified by:

getConfFile in interface Job

Returns:

a path to where the config file for this job is located.

getName

public String getName()

Specified by:

getName in interface Job

getTotalMaps

public int getTotalMaps()

Specified by:

getTotalMaps in interface Job

getTotalReduces

public int getTotalReduces()

Specified by:

getTotalReduces in interface Job

getJobACLs

public Map<org.apache.hadoop.mapreduce.JobACL,org.apache.hadoop.security.authorize.AccessControlList> getJobACLs()

Specified by:

getJobACLs in interface Job

Returns:

the ACLs for this job for each type of JobACL given.

getAMInfos

public List<org.apache.hadoop.mapreduce.v2.api.records.AMInfo> getAMInfos()

Specified by:

getAMInfos in interface Job

Returns:

information for MR AppMasters (previously failed and current)

escapeString

public static String escapeString(String data)

constructFinalFullcounters

@InterfaceAudience.Private
public void constructFinalFullcounters()

addDiagnostic

protected void addDiagnostic(String diag)

loadConfFile

public org.apache.hadoop.conf.Configuration loadConfFile()
                                                  throws IOException

Specified by:

loadConfFile in interface Job

Returns:

a parsed version of the config files pointed to by Job.getConfFile().

Throws:

IOException - on any error trying to load the conf file.