@InterfaceAudience.Private @InterfaceStability.Unstable public abstract class AbstractYarnScheduler<T extends SchedulerApplicationAttempt,N extends SchedulerNode> extends org.apache.hadoop.service.AbstractService implements ResourceScheduler
Modifier and Type | Field and Description |
---|---|
protected ConcurrentMap<org.apache.hadoop.yarn.api.records.ApplicationId,SchedulerApplication<T>> |
applications |
protected org.apache.hadoop.yarn.api.records.Resource |
clusterResource |
protected static Allocation |
EMPTY_ALLOCATION |
protected static List<org.apache.hadoop.yarn.api.records.Container> |
EMPTY_CONTAINER_LIST |
protected org.apache.hadoop.yarn.api.records.Resource |
minimumAllocation |
protected int |
nmExpireInterval |
protected Map<org.apache.hadoop.yarn.api.records.NodeId,N> |
nodes |
protected RMContext |
rmContext |
Constructor and Description |
---|
AbstractYarnScheduler(String name)
Construct the service.
|
Modifier and Type | Method and Description |
---|---|
void |
addBlacklistedNodeIdsToList(SchedulerApplicationAttempt app,
List<org.apache.hadoop.yarn.api.records.NodeId> blacklistNodeIdList)
Add blacklisted NodeIds to the list that is passed.
|
void |
addQueue(Queue newQueue)
Add to the scheduler a new Queue.
|
protected abstract void |
completedContainer(RMContainer rmContainer,
org.apache.hadoop.yarn.api.records.ContainerStatus containerStatus,
RMContainerEventType event) |
protected void |
containerLaunchedOnNode(org.apache.hadoop.yarn.api.records.ContainerId containerId,
SchedulerNode node) |
protected void |
createReleaseCache() |
T |
getApplicationAttempt(org.apache.hadoop.yarn.api.records.ApplicationAttemptId applicationAttemptId) |
org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport |
getAppResourceUsageReport(org.apache.hadoop.yarn.api.records.ApplicationAttemptId appAttemptId)
Get a resource usage report from a given app attempt ID.
|
org.apache.hadoop.yarn.api.records.Resource |
getClusterResource()
Get the whole resource capacity of the cluster.
|
T |
getCurrentAttemptForContainer(org.apache.hadoop.yarn.api.records.ContainerId containerId) |
DebugController |
getDebugController() |
org.apache.hadoop.yarn.api.records.Resource |
getMaximumResourceCapability()
Get maximum allocatable
Resource at the cluster level. |
org.apache.hadoop.yarn.api.records.Resource |
getMaximumResourceCapability(String queueName)
Get maximum allocatable
Resource for the queue specified. |
org.apache.hadoop.yarn.api.records.Resource |
getMinimumResourceCapability()
Get minimum allocatable
Resource . |
SchedulerNodeReport |
getNodeReport(org.apache.hadoop.yarn.api.records.NodeId nodeId)
Get node resource usage report.
|
List<org.apache.hadoop.yarn.api.records.ResourceRequest> |
getPendingResourceRequestsForAttempt(org.apache.hadoop.yarn.api.records.ApplicationAttemptId attemptId) |
Set<String> |
getPlanQueues()
Gets the list of names for queues managed by the Reservation System
|
RMContainer |
getRMContainer(org.apache.hadoop.yarn.api.records.ContainerId containerId)
Get the container for the given containerId.
|
SchedulerAppReport |
getSchedulerAppInfo(org.apache.hadoop.yarn.api.records.ApplicationAttemptId appAttemptId)
Get the Scheduler app for a given app attempt Id.
|
Map<org.apache.hadoop.yarn.api.records.ApplicationId,SchedulerApplication<T>> |
getSchedulerApplications() |
SchedulerNode |
getSchedulerNode(org.apache.hadoop.yarn.api.records.NodeId nodeId) |
EnumSet<org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes> |
getSchedulingResourceTypes()
Return a collection of the resource types that are considered when
scheduling
|
List<org.apache.hadoop.yarn.api.records.Container> |
getTransferredContainers(org.apache.hadoop.yarn.api.records.ApplicationAttemptId currentAttempt) |
protected void |
initMaximumResourceCapability(org.apache.hadoop.yarn.api.records.Resource maximumAllocation) |
void |
killAllAppsInQueue(String queueName)
Terminate all applications in the specified queue.
|
void |
moveAllApps(String sourceQueue,
String destQueue)
Completely drain sourceQueue of applications, by moving all of them to
destQueue.
|
String |
moveApplication(org.apache.hadoop.yarn.api.records.ApplicationId appId,
String newQueue)
Moves the given application to the given queue
|
void |
recoverContainersOnNode(List<org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus> containerReports,
RMNode nm) |
protected void |
recoverResourceRequestForContainer(RMContainer rmContainer)
Recover resource request back from RMContainer when a container is
preempted before AM pulled the same.
|
protected void |
refreshMaximumAllocation(org.apache.hadoop.yarn.api.records.Resource newMaxAlloc) |
protected void |
releaseContainers(List<org.apache.hadoop.yarn.api.records.ContainerId> containers,
SchedulerApplicationAttempt attempt) |
void |
removeQueue(String queueName)
Remove an existing queue.
|
void |
serviceInit(org.apache.hadoop.conf.Configuration conf) |
void |
setEntitlement(String queue,
QueueEntitlement entitlement)
This method increase the entitlement for current queue (must respect
invariants, e.g., no overcommit of parents, non negative, etc.).
|
protected void |
updateMaximumAllocation(SchedulerNode node,
boolean add) |
void |
updateNodeResource(RMNode nm,
org.apache.hadoop.yarn.api.records.ResourceOption resourceOption)
Process resource update on a node.
|
close, getBlockers, getConfig, getFailureCause, getFailureState, getLifecycleHistory, getName, getServiceState, getStartTime, init, isInState, noteFailure, putBlocker, registerGlobalListener, registerServiceListener, removeBlocker, serviceStart, serviceStop, setConfig, start, stop, toString, unregisterGlobalListener, unregisterServiceListener, waitForServiceToStop
clone, equals, finalize, getClass, hashCode, notify, notifyAll, wait, wait, wait
reinitialize, setRMContext
allocate, checkAccess, getAppsInQueue, getIncrementResourceCapability, getNumClusterNodes, getQueueInfo, getQueueUserAclInfo, getResourceCalculator, getRootQueueMetrics
recover
protected Map<org.apache.hadoop.yarn.api.records.NodeId,N extends SchedulerNode> nodes
protected org.apache.hadoop.yarn.api.records.Resource clusterResource
protected org.apache.hadoop.yarn.api.records.Resource minimumAllocation
protected RMContext rmContext
protected ConcurrentMap<org.apache.hadoop.yarn.api.records.ApplicationId,SchedulerApplication<T extends SchedulerApplicationAttempt>> applications
protected int nmExpireInterval
protected static final List<org.apache.hadoop.yarn.api.records.Container> EMPTY_CONTAINER_LIST
protected static final Allocation EMPTY_ALLOCATION
public AbstractYarnScheduler(String name)
name
- service namepublic void serviceInit(org.apache.hadoop.conf.Configuration conf) throws Exception
serviceInit
in class org.apache.hadoop.service.AbstractService
Exception
public List<org.apache.hadoop.yarn.api.records.Container> getTransferredContainers(org.apache.hadoop.yarn.api.records.ApplicationAttemptId currentAttempt)
public Map<org.apache.hadoop.yarn.api.records.ApplicationId,SchedulerApplication<T>> getSchedulerApplications()
public void addBlacklistedNodeIdsToList(SchedulerApplicationAttempt app, List<org.apache.hadoop.yarn.api.records.NodeId> blacklistNodeIdList)
app
- application attempt.blacklistNodeIdList
- the list to store blacklisted NodeIds.public org.apache.hadoop.yarn.api.records.Resource getClusterResource()
YarnScheduler
getClusterResource
in interface YarnScheduler
public org.apache.hadoop.yarn.api.records.Resource getMinimumResourceCapability()
YarnScheduler
Resource
.getMinimumResourceCapability
in interface YarnScheduler
public org.apache.hadoop.yarn.api.records.Resource getMaximumResourceCapability()
YarnScheduler
Resource
at the cluster level.getMaximumResourceCapability
in interface YarnScheduler
public org.apache.hadoop.yarn.api.records.Resource getMaximumResourceCapability(String queueName)
YarnScheduler
Resource
for the queue specified.getMaximumResourceCapability
in interface YarnScheduler
queueName
- queue nameprotected void initMaximumResourceCapability(org.apache.hadoop.yarn.api.records.Resource maximumAllocation)
protected void containerLaunchedOnNode(org.apache.hadoop.yarn.api.records.ContainerId containerId, SchedulerNode node)
public T getApplicationAttempt(org.apache.hadoop.yarn.api.records.ApplicationAttemptId applicationAttemptId)
public SchedulerAppReport getSchedulerAppInfo(org.apache.hadoop.yarn.api.records.ApplicationAttemptId appAttemptId)
YarnScheduler
getSchedulerAppInfo
in interface YarnScheduler
appAttemptId
- the id of the application attemptpublic org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport getAppResourceUsageReport(org.apache.hadoop.yarn.api.records.ApplicationAttemptId appAttemptId)
YarnScheduler
getAppResourceUsageReport
in interface YarnScheduler
appAttemptId
- the id of the application attemptpublic T getCurrentAttemptForContainer(org.apache.hadoop.yarn.api.records.ContainerId containerId)
public RMContainer getRMContainer(org.apache.hadoop.yarn.api.records.ContainerId containerId)
YarnScheduler
getRMContainer
in interface YarnScheduler
public SchedulerNodeReport getNodeReport(org.apache.hadoop.yarn.api.records.NodeId nodeId)
YarnScheduler
getNodeReport
in interface YarnScheduler
SchedulerNodeReport
for the node or null
if nodeId does not point to a defined node.public String moveApplication(org.apache.hadoop.yarn.api.records.ApplicationId appId, String newQueue) throws org.apache.hadoop.yarn.exceptions.YarnException
YarnScheduler
moveApplication
in interface YarnScheduler
org.apache.hadoop.yarn.exceptions.YarnException
- if the move cannot be carried outpublic void removeQueue(String queueName) throws org.apache.hadoop.yarn.exceptions.YarnException
YarnScheduler
removeQueue
in interface YarnScheduler
queueName
- name of the queue to removeorg.apache.hadoop.yarn.exceptions.YarnException
public void addQueue(Queue newQueue) throws org.apache.hadoop.yarn.exceptions.YarnException
YarnScheduler
addQueue
in interface YarnScheduler
newQueue
- the queue being added.org.apache.hadoop.yarn.exceptions.YarnException
public void setEntitlement(String queue, QueueEntitlement entitlement) throws org.apache.hadoop.yarn.exceptions.YarnException
YarnScheduler
setEntitlement
in interface YarnScheduler
queue
- the queue for which we change entitlemententitlement
- the new entitlement for the queue (capacity,
maxCapacity, etc..)org.apache.hadoop.yarn.exceptions.YarnException
public void recoverContainersOnNode(List<org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus> containerReports, RMNode nm)
protected void recoverResourceRequestForContainer(RMContainer rmContainer)
rmContainer
- protected void createReleaseCache()
protected abstract void completedContainer(RMContainer rmContainer, org.apache.hadoop.yarn.api.records.ContainerStatus containerStatus, RMContainerEventType event)
protected void releaseContainers(List<org.apache.hadoop.yarn.api.records.ContainerId> containers, SchedulerApplicationAttempt attempt)
public SchedulerNode getSchedulerNode(org.apache.hadoop.yarn.api.records.NodeId nodeId)
public void moveAllApps(String sourceQueue, String destQueue) throws org.apache.hadoop.yarn.exceptions.YarnException
YarnScheduler
moveAllApps
in interface YarnScheduler
org.apache.hadoop.yarn.exceptions.YarnException
public void killAllAppsInQueue(String queueName) throws org.apache.hadoop.yarn.exceptions.YarnException
YarnScheduler
killAllAppsInQueue
in interface YarnScheduler
queueName
- the name of queue to be drainedorg.apache.hadoop.yarn.exceptions.YarnException
public void updateNodeResource(RMNode nm, org.apache.hadoop.yarn.api.records.ResourceOption resourceOption)
public EnumSet<org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes> getSchedulingResourceTypes()
getSchedulingResourceTypes
in interface YarnScheduler
public Set<String> getPlanQueues() throws org.apache.hadoop.yarn.exceptions.YarnException
YarnScheduler
getPlanQueues
in interface YarnScheduler
org.apache.hadoop.yarn.exceptions.YarnException
protected void updateMaximumAllocation(SchedulerNode node, boolean add)
protected void refreshMaximumAllocation(org.apache.hadoop.yarn.api.records.Resource newMaxAlloc)
public List<org.apache.hadoop.yarn.api.records.ResourceRequest> getPendingResourceRequestsForAttempt(org.apache.hadoop.yarn.api.records.ApplicationAttemptId attemptId)
public DebugController getDebugController()
Copyright © 2019 Apache Software Foundation. All Rights Reserved.