Package org.apache.hadoop.yarn.server.resourcemanager.scheduler.distributed

Class NodeQueueLoadMonitor

java.lang.Object

org.apache.hadoop.yarn.server.resourcemanager.scheduler.distributed.NodeQueueLoadMonitor

All Implemented Interfaces:

ClusterMonitor

public class NodeQueueLoadMonitor
extends java.lang.Object
implements ClusterMonitor

The NodeQueueLoadMonitor keeps track of load metrics (such as queue length and total wait time) associated with Container Queues on the Node Manager. It uses this information to periodically sort the Nodes from least to most loaded.

Nested Class Summary

Nested Classes

Modifier and Type	Class	Description
static class	NodeQueueLoadMonitor.LoadComparator	The comparator used to specify the metric against which the load of two Nodes are compared.

Field Summary

Fields

Modifier and Type	Field	Description
protected java.util.Map<org.apache.hadoop.yarn.api.records.NodeId,ClusterNode>	clusterNodes
protected java.util.concurrent.locks.ReentrantReadWriteLock	clusterNodesLock
protected NodeQueueLoadMonitor.LoadComparator	comparator
protected static org.slf4j.Logger	LOG
protected java.util.Map<java.lang.String,RMNode>	nodeByHostName
protected java.util.Map<java.lang.String,java.util.Set<org.apache.hadoop.yarn.api.records.NodeId>>	nodeIdsByRack
protected int	numNodesForAnyAllocation
protected java.util.List<org.apache.hadoop.yarn.api.records.NodeId>	sortedNodes
protected java.util.concurrent.locks.ReentrantReadWriteLock	sortedNodesLock
protected QueueLimitCalculator	thresholdCalculator

Constructor Summary

Constructors

Constructor	Description
NodeQueueLoadMonitor(long nodeComputationInterval, NodeQueueLoadMonitor.LoadComparator comparator, int numNodes)

Method Summary

Modifier and Type	Method	Description
protected void	addIntoNodeIdsByRack(RMNode addedNode)
void	addNode(java.util.List<org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus> containerStatuses, RMNode rmNode)
protected java.util.List<org.apache.hadoop.yarn.api.records.NodeId>	getCandidatesForSelectAnyNode()
QueueLimitCalculator	getThresholdCalculator()
void	initThresholdCalculator(float sigma, int limitMin, int limitMax)
protected void	onExistingNodeUpdated(RMNode rmNode, ClusterNode clusterNode, org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus status)
protected void	onNewNodeAdded(RMNode rmNode, org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus status)
protected void	onNodeRemoved(ClusterNode node)	Provide an integration point for extended class
protected void	removeFromNodeIdsByRack(RMNode removedNode)
void	removeNode(RMNode removedRMNode)
RMNode	selectAnyNode(java.util.Set<java.lang.String> blacklist, org.apache.hadoop.yarn.api.records.Resource request)	Selects a node from all ClusterNodes for resource allocation, excluding blacklisted nodes.
java.util.List<org.apache.hadoop.yarn.api.records.NodeId>	selectLeastLoadedNodes(int k)	Returns 'K' of the least Loaded Node Ids as ordered list.
RMNode	selectLocalNode(java.lang.String hostName, java.util.Set<java.lang.String> blacklist, org.apache.hadoop.yarn.api.records.Resource request)	Selects the node as specified by hostName for resource allocation, unless the node has been blacklisted.
java.util.List<org.apache.hadoop.yarn.api.records.NodeId>	selectNodes()	Returns all Node Ids as ordered list from Least to Most Loaded.
RMNode	selectRackLocalNode(java.lang.String rackName, java.util.Set<java.lang.String> blacklist, org.apache.hadoop.yarn.api.records.Resource request)	Selects a node from the rack as specified by rackName for resource allocation, excluding blacklisted nodes
protected java.util.List<ClusterNode>	sortNodes(boolean excludeFullNodes)
void	stop()
void	updateNode(RMNode rmNode)
void	updateNodeResource(RMNode rmNode, org.apache.hadoop.yarn.api.records.ResourceOption resourceOption)
protected void	updateSortedNodes()

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

LOG

protected static final org.slf4j.Logger LOG

numNodesForAnyAllocation

protected int numNodesForAnyAllocation

sortedNodes

protected final java.util.List<org.apache.hadoop.yarn.api.records.NodeId> sortedNodes

clusterNodes

protected final java.util.Map<org.apache.hadoop.yarn.api.records.NodeId,ClusterNode> clusterNodes

nodeByHostName

protected final java.util.Map<java.lang.String,RMNode> nodeByHostName

nodeIdsByRack

protected final java.util.Map<java.lang.String,java.util.Set<org.apache.hadoop.yarn.api.records.NodeId>> nodeIdsByRack

comparator

protected final NodeQueueLoadMonitor.LoadComparator comparator

thresholdCalculator

protected QueueLimitCalculator thresholdCalculator

sortedNodesLock

protected java.util.concurrent.locks.ReentrantReadWriteLock sortedNodesLock

clusterNodesLock

protected java.util.concurrent.locks.ReentrantReadWriteLock clusterNodesLock

Constructor Detail

NodeQueueLoadMonitor

public NodeQueueLoadMonitor(long nodeComputationInterval,
                            NodeQueueLoadMonitor.LoadComparator comparator,
                            int numNodes)

Method Detail

updateSortedNodes

protected void updateSortedNodes()

getThresholdCalculator

public QueueLimitCalculator getThresholdCalculator()

stop

public void stop()

initThresholdCalculator

public void initThresholdCalculator(float sigma,
                                    int limitMin,
                                    int limitMax)

addNode

public void addNode(java.util.List<org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus> containerStatuses,
                    RMNode rmNode)

Specified by:

addNode in interface ClusterMonitor

removeNode

public void removeNode(RMNode removedRMNode)

Specified by:

removeNode in interface ClusterMonitor

onNodeRemoved

protected void onNodeRemoved(ClusterNode node)

Provide an integration point for extended class

Parameters:

node - the node removed

updateNode

public void updateNode(RMNode rmNode)

Specified by:

updateNode in interface ClusterMonitor

onNewNodeAdded

protected void onNewNodeAdded(RMNode rmNode,
                              org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus status)

onExistingNodeUpdated

protected void onExistingNodeUpdated(RMNode rmNode,
                                     ClusterNode clusterNode,
                                     org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus status)

updateNodeResource

public void updateNodeResource(RMNode rmNode,
                               org.apache.hadoop.yarn.api.records.ResourceOption resourceOption)

Specified by:

updateNodeResource in interface ClusterMonitor

selectNodes

public java.util.List<org.apache.hadoop.yarn.api.records.NodeId> selectNodes()

Returns all Node Ids as ordered list from Least to Most Loaded.

Returns:

ordered list of nodes

selectLeastLoadedNodes

public java.util.List<org.apache.hadoop.yarn.api.records.NodeId> selectLeastLoadedNodes(int k)

Returns 'K' of the least Loaded Node Ids as ordered list.

Parameters:

k - max number of nodes to return

Returns:

ordered list of nodes

selectLocalNode

public RMNode selectLocalNode(java.lang.String hostName,
                              java.util.Set<java.lang.String> blacklist,
                              org.apache.hadoop.yarn.api.records.Resource request)

Selects the node as specified by hostName for resource allocation, unless the node has been blacklisted.

Parameters:

hostName - the hostname of the node for local resource allocation

blacklist - the blacklisted nodes

request - the requested resource

Returns:

the selected node, null if the node is full or is blacklisted

selectRackLocalNode

public RMNode selectRackLocalNode(java.lang.String rackName,
                                  java.util.Set<java.lang.String> blacklist,
                                  org.apache.hadoop.yarn.api.records.Resource request)

Selects a node from the rack as specified by rackName for resource allocation, excluding blacklisted nodes

Parameters:

rackName - the rack name for rack-local resource allocation

blacklist - the blacklisted nodes

request - the requested resource

Returns:

the selected node, null if no suitable nodes

selectAnyNode

public RMNode selectAnyNode(java.util.Set<java.lang.String> blacklist,
                            org.apache.hadoop.yarn.api.records.Resource request)

Selects a node from all ClusterNodes for resource allocation, excluding blacklisted nodes.

Parameters:

blacklist - the blacklisted nodes

request - the requested resource

Returns:

the selected node, null if no suitable nodes

getCandidatesForSelectAnyNode

protected java.util.List<org.apache.hadoop.yarn.api.records.NodeId> getCandidatesForSelectAnyNode()

removeFromNodeIdsByRack

protected void removeFromNodeIdsByRack(RMNode removedNode)

addIntoNodeIdsByRack

protected void addIntoNodeIdsByRack(RMNode addedNode)

sortNodes

protected java.util.List<ClusterNode> sortNodes(boolean excludeFullNodes)