@InterfaceAudience.LimitedPrivate(value="yarn") @InterfaceStability.Evolving public class CapacityScheduler extends AbstractYarnScheduler<FiCaSchedulerApp,FiCaSchedulerNode> implements PreemptableResourceScheduler, CapacitySchedulerContext, org.apache.hadoop.conf.Configurable
Modifier and Type | Field and Description |
---|---|
static String |
ROOT_QUEUE |
protected long |
THREAD_JOIN_TIMEOUT_MS |
applications, clusterResource, EMPTY_ALLOCATION, EMPTY_CONTAINER_LIST, minimumAllocation, nmExpireInterval, nodes, rmContext
Constructor and Description |
---|
CapacityScheduler() |
Modifier and Type | Method and Description |
---|---|
void |
addQueue(Queue queue)
Add to the scheduler a new Queue.
|
Allocation |
allocate(org.apache.hadoop.yarn.api.records.ApplicationAttemptId applicationAttemptId,
List<org.apache.hadoop.yarn.api.records.ResourceRequest> ask,
List<org.apache.hadoop.yarn.api.records.ContainerId> release,
List<String> blacklistAdditions,
List<String> blacklistRemovals)
The main api between the ApplicationMaster and the Scheduler.
|
boolean |
checkAccess(org.apache.hadoop.security.UserGroupInformation callerUGI,
org.apache.hadoop.yarn.api.records.QueueACL acl,
String queueName)
Check if the user has permission to perform the operation.
|
protected void |
completedContainer(RMContainer rmContainer,
org.apache.hadoop.yarn.api.records.ContainerStatus containerStatus,
RMContainerEventType event) |
void |
dropContainerReservation(RMContainer container)
If the scheduler support container reservations, this method is used to
ask the scheduler to drop the reservation for the given container.
|
FiCaSchedulerApp |
getApplicationAttempt(org.apache.hadoop.yarn.api.records.ApplicationAttemptId applicationAttemptId) |
Comparator<FiCaSchedulerApp> |
getApplicationComparator() |
List<org.apache.hadoop.yarn.api.records.ApplicationAttemptId> |
getAppsInQueue(String queueName)
Gets the apps under a given queue
|
org.apache.hadoop.conf.Configuration |
getConf()
Get the yarn configuration.
|
CapacitySchedulerConfiguration |
getConfiguration() |
RMContainerTokenSecretManager |
getContainerTokenSecretManager() |
org.apache.hadoop.yarn.api.records.Resource |
getIncrementResourceCapability()
Get increment allocatable
Resource . |
String |
getMappedQueueForTest(String user) |
org.apache.hadoop.yarn.api.records.Resource |
getMaximumResourceCapability(String queueName)
Get maximum allocatable
Resource for the queue specified. |
FiCaSchedulerNode |
getNode(org.apache.hadoop.yarn.api.records.NodeId nodeId) |
int |
getNumClusterNodes()
Get the number of nodes available in the cluster.
|
Set<String> |
getPlanQueues()
Gets the list of names for queues managed by the Reservation System
|
CSQueue |
getQueue(String queueName) |
Comparator<CSQueue> |
getQueueComparator() |
org.apache.hadoop.yarn.api.records.QueueInfo |
getQueueInfo(String queueName,
boolean includeChildQueues,
boolean recursive)
Get queue information
|
List<org.apache.hadoop.yarn.api.records.QueueUserACLInfo> |
getQueueUserAclInfo()
Get acls for queues for current user.
|
org.apache.hadoop.yarn.util.resource.ResourceCalculator |
getResourceCalculator() |
RMContext |
getRMContext() |
CSQueue |
getRootQueue() |
QueueMetrics |
getRootQueueMetrics()
Get the root queue for the scheduler.
|
EnumSet<org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes> |
getSchedulingResourceTypes()
Return a collection of the resource types that are considered when
scheduling
|
void |
handle(SchedulerEvent event) |
void |
killContainer(RMContainer cont)
Ask the scheduler to forcibly interrupt the container given as input
|
String |
moveApplication(org.apache.hadoop.yarn.api.records.ApplicationId appId,
String targetQueueName)
Moves the given application to the given queue
|
void |
preemptContainer(org.apache.hadoop.yarn.api.records.ApplicationAttemptId aid,
RMContainer cont)
Ask the scheduler to obtain back the container from a specific application
by issuing a preemption request
|
void |
recover(RMStateStore.RMState state) |
void |
reinitialize(org.apache.hadoop.conf.Configuration conf,
RMContext rmContext)
Re-initialize the
ResourceScheduler . |
void |
removeQueue(String queueName)
Remove an existing queue.
|
void |
serviceInit(org.apache.hadoop.conf.Configuration conf) |
void |
serviceStart() |
void |
serviceStop() |
void |
setConf(org.apache.hadoop.conf.Configuration conf) |
void |
setEntitlement(String inQueue,
QueueEntitlement entitlement)
This method increase the entitlement for current queue (must respect
invariants, e.g., no overcommit of parents, non negative, etc.).
|
static void |
setQueueAcls(org.apache.hadoop.yarn.security.YarnAuthorizationProvider authorizer,
Map<String,CSQueue> queues) |
void |
setRMContext(RMContext rmContext)
Set RMContext for
ResourceScheduler . |
addBlacklistedNodeIdsToList, containerLaunchedOnNode, createReleaseCache, getAppResourceUsageReport, getClusterResource, getCurrentAttemptForContainer, getDebugController, getMaximumResourceCapability, getMinimumResourceCapability, getNodeReport, getPendingResourceRequestsForAttempt, getRMContainer, getSchedulerAppInfo, getSchedulerApplications, getSchedulerNode, getTransferredContainers, initMaximumResourceCapability, killAllAppsInQueue, moveAllApps, recoverContainersOnNode, recoverResourceRequestForContainer, refreshMaximumAllocation, releaseContainers, updateMaximumAllocation, updateNodeResource
close, getBlockers, getConfig, getFailureCause, getFailureState, getLifecycleHistory, getName, getServiceState, getStartTime, init, isInState, noteFailure, putBlocker, registerGlobalListener, registerServiceListener, removeBlocker, setConfig, start, stop, toString, unregisterGlobalListener, unregisterServiceListener, waitForServiceToStop
clone, equals, finalize, getClass, hashCode, notify, notifyAll, wait, wait, wait
getAppResourceUsageReport, getClusterResource, getMaximumResourceCapability, getMinimumResourceCapability, getNodeReport, getRMContainer, getSchedulerAppInfo, killAllAppsInQueue, moveAllApps
getClusterResource, getMaximumResourceCapability, getMinimumResourceCapability
protected final long THREAD_JOIN_TIMEOUT_MS
@InterfaceAudience.Private public static final String ROOT_QUEUE
public void setConf(org.apache.hadoop.conf.Configuration conf)
setConf
in interface org.apache.hadoop.conf.Configurable
public org.apache.hadoop.conf.Configuration getConf()
CapacitySchedulerContext
getConf
in interface org.apache.hadoop.conf.Configurable
getConf
in interface CapacitySchedulerContext
public String getMappedQueueForTest(String user) throws IOException
IOException
public QueueMetrics getRootQueueMetrics()
YarnScheduler
getRootQueueMetrics
in interface YarnScheduler
public CSQueue getRootQueue()
public CapacitySchedulerConfiguration getConfiguration()
getConfiguration
in interface CapacitySchedulerContext
public RMContainerTokenSecretManager getContainerTokenSecretManager()
getContainerTokenSecretManager
in interface CapacitySchedulerContext
public Comparator<FiCaSchedulerApp> getApplicationComparator()
getApplicationComparator
in interface CapacitySchedulerContext
public org.apache.hadoop.yarn.util.resource.ResourceCalculator getResourceCalculator()
getResourceCalculator
in interface CapacitySchedulerContext
getResourceCalculator
in interface YarnScheduler
public Comparator<CSQueue> getQueueComparator()
getQueueComparator
in interface CapacitySchedulerContext
public int getNumClusterNodes()
YarnScheduler
getNumClusterNodes
in interface CapacitySchedulerContext
getNumClusterNodes
in interface YarnScheduler
public RMContext getRMContext()
getRMContext
in interface CapacitySchedulerContext
public void setRMContext(RMContext rmContext)
ResourceScheduler
ResourceScheduler
.
This method should be called immediately after instantiating
a scheduler once.setRMContext
in interface ResourceScheduler
rmContext
- created by ResourceManagerpublic void serviceInit(org.apache.hadoop.conf.Configuration conf) throws Exception
serviceInit
in class AbstractYarnScheduler<FiCaSchedulerApp,FiCaSchedulerNode>
Exception
public void serviceStart() throws Exception
serviceStart
in class org.apache.hadoop.service.AbstractService
Exception
public void serviceStop() throws Exception
serviceStop
in class org.apache.hadoop.service.AbstractService
Exception
public void reinitialize(org.apache.hadoop.conf.Configuration conf, RMContext rmContext) throws IOException
ResourceScheduler
ResourceScheduler
.reinitialize
in interface ResourceScheduler
conf
- configurationIOException
public static void setQueueAcls(org.apache.hadoop.yarn.security.YarnAuthorizationProvider authorizer, Map<String,CSQueue> queues) throws IOException
IOException
@Lock(value=org.apache.hadoop.yarn.server.utils.Lock.NoLock.class) public Allocation allocate(org.apache.hadoop.yarn.api.records.ApplicationAttemptId applicationAttemptId, List<org.apache.hadoop.yarn.api.records.ResourceRequest> ask, List<org.apache.hadoop.yarn.api.records.ContainerId> release, List<String> blacklistAdditions, List<String> blacklistRemovals)
YarnScheduler
allocate
in interface YarnScheduler
Allocation
for the application@Lock(value=org.apache.hadoop.yarn.server.utils.Lock.NoLock.class) public org.apache.hadoop.yarn.api.records.QueueInfo getQueueInfo(String queueName, boolean includeChildQueues, boolean recursive) throws IOException
YarnScheduler
getQueueInfo
in interface YarnScheduler
queueName
- queue nameincludeChildQueues
- include child queues?recursive
- get children queues?IOException
@Lock(value=org.apache.hadoop.yarn.server.utils.Lock.NoLock.class) public List<org.apache.hadoop.yarn.api.records.QueueUserACLInfo> getQueueUserAclInfo()
YarnScheduler
getQueueUserAclInfo
in interface YarnScheduler
public org.apache.hadoop.yarn.api.records.Resource getIncrementResourceCapability()
YarnScheduler
Resource
.getIncrementResourceCapability
in interface YarnScheduler
public void handle(SchedulerEvent event)
handle
in interface org.apache.hadoop.yarn.event.EventHandler<SchedulerEvent>
@Lock(value=CapacityScheduler.class) protected void completedContainer(RMContainer rmContainer, org.apache.hadoop.yarn.api.records.ContainerStatus containerStatus, RMContainerEventType event)
completedContainer
in class AbstractYarnScheduler<FiCaSchedulerApp,FiCaSchedulerNode>
@Lock(value=org.apache.hadoop.yarn.server.utils.Lock.NoLock.class) public FiCaSchedulerApp getApplicationAttempt(org.apache.hadoop.yarn.api.records.ApplicationAttemptId applicationAttemptId)
getApplicationAttempt
in class AbstractYarnScheduler<FiCaSchedulerApp,FiCaSchedulerNode>
@Lock(value=org.apache.hadoop.yarn.server.utils.Lock.NoLock.class) public FiCaSchedulerNode getNode(org.apache.hadoop.yarn.api.records.NodeId nodeId)
getNode
in interface CapacitySchedulerContext
@Lock(value=org.apache.hadoop.yarn.server.utils.Lock.NoLock.class) public void recover(RMStateStore.RMState state) throws Exception
recover
in interface Recoverable
Exception
public void dropContainerReservation(RMContainer container)
PreemptableResourceScheduler
dropContainerReservation
in interface PreemptableResourceScheduler
container
- Reference to reserved container allocation.public void preemptContainer(org.apache.hadoop.yarn.api.records.ApplicationAttemptId aid, RMContainer cont)
PreemptableResourceScheduler
preemptContainer
in interface PreemptableResourceScheduler
aid
- the application from which we want to get a container backcont
- the container we want backpublic void killContainer(RMContainer cont)
PreemptableResourceScheduler
killContainer
in interface PreemptableResourceScheduler
public boolean checkAccess(org.apache.hadoop.security.UserGroupInformation callerUGI, org.apache.hadoop.yarn.api.records.QueueACL acl, String queueName)
YarnScheduler
QueueACL.ADMINISTER_QUEUE
permission,
this user can view/modify the applications in this queuecheckAccess
in interface YarnScheduler
true
if the user has the permission,
false
otherwisepublic List<org.apache.hadoop.yarn.api.records.ApplicationAttemptId> getAppsInQueue(String queueName)
YarnScheduler
getAppsInQueue
in interface YarnScheduler
queueName
- the name of the queue.public void removeQueue(String queueName) throws SchedulerDynamicEditException
YarnScheduler
removeQueue
in interface YarnScheduler
removeQueue
in class AbstractYarnScheduler<FiCaSchedulerApp,FiCaSchedulerNode>
queueName
- name of the queue to removeSchedulerDynamicEditException
public void addQueue(Queue queue) throws SchedulerDynamicEditException
YarnScheduler
addQueue
in interface YarnScheduler
addQueue
in class AbstractYarnScheduler<FiCaSchedulerApp,FiCaSchedulerNode>
queue
- the queue being added.SchedulerDynamicEditException
public void setEntitlement(String inQueue, QueueEntitlement entitlement) throws SchedulerDynamicEditException, org.apache.hadoop.yarn.exceptions.YarnException
YarnScheduler
setEntitlement
in interface YarnScheduler
setEntitlement
in class AbstractYarnScheduler<FiCaSchedulerApp,FiCaSchedulerNode>
inQueue
- the queue for which we change entitlemententitlement
- the new entitlement for the queue (capacity,
maxCapacity, etc..)org.apache.hadoop.yarn.exceptions.YarnException
SchedulerDynamicEditException
public String moveApplication(org.apache.hadoop.yarn.api.records.ApplicationId appId, String targetQueueName) throws org.apache.hadoop.yarn.exceptions.YarnException
YarnScheduler
moveApplication
in interface YarnScheduler
moveApplication
in class AbstractYarnScheduler<FiCaSchedulerApp,FiCaSchedulerNode>
org.apache.hadoop.yarn.exceptions.YarnException
- if the move cannot be carried outpublic EnumSet<org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes> getSchedulingResourceTypes()
getSchedulingResourceTypes
in interface YarnScheduler
getSchedulingResourceTypes
in class AbstractYarnScheduler<FiCaSchedulerApp,FiCaSchedulerNode>
public org.apache.hadoop.yarn.api.records.Resource getMaximumResourceCapability(String queueName)
YarnScheduler
Resource
for the queue specified.getMaximumResourceCapability
in interface CapacitySchedulerContext
getMaximumResourceCapability
in interface YarnScheduler
getMaximumResourceCapability
in class AbstractYarnScheduler<FiCaSchedulerApp,FiCaSchedulerNode>
queueName
- queue namepublic Set<String> getPlanQueues()
YarnScheduler
getPlanQueues
in interface YarnScheduler
getPlanQueues
in class AbstractYarnScheduler<FiCaSchedulerApp,FiCaSchedulerNode>
Copyright © 2019 Apache Software Foundation. All Rights Reserved.