Package org.apache.hadoop.yarn.nodelabels

Class CommonNodeLabelsManager

java.lang.Object

org.apache.hadoop.service.AbstractService

org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager

All Implemented Interfaces:

java.io.Closeable, java.lang.AutoCloseable, org.apache.hadoop.service.Service

@Private
public class CommonNodeLabelsManager
extends org.apache.hadoop.service.AbstractService

Nested Class Summary

Nested Classes

Modifier and Type	Class	Description
static class	CommonNodeLabelsManager.Host	A Host can have multiple Nodes
protected static class	CommonNodeLabelsManager.Node

Nested classes/interfaces inherited from interface org.apache.hadoop.service.Service

org.apache.hadoop.service.Service.STATE

Field Summary

Fields

Modifier and Type	Field	Description
static java.util.Set<java.lang.String>	ACCESS_ANY_LABEL_SET
static java.lang.String	ANY
protected Dispatcher	dispatcher
static java.util.Set<org.apache.hadoop.yarn.api.records.NodeLabel>	EMPTY_NODELABEL_SET
static java.util.Set<java.lang.String>	EMPTY_STRING_SET
protected java.util.concurrent.ConcurrentMap<java.lang.String,RMNodeLabel>	labelCollections
protected static org.slf4j.Logger	LOG
static java.lang.String	NO_LABEL	If a user doesn't specify label of a queue or node, it belongs DEFAULT_LABEL
static java.lang.String	NODE_LABELS_NOT_ENABLED_ERR	Error messages
protected java.util.concurrent.ConcurrentMap<java.lang.String,CommonNodeLabelsManager.Host>	nodeCollections
protected RMNodeLabel	noNodeLabel
protected java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock	readLock
protected NodeLabelsStore	store
static int	WILDCARD_PORT
protected java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock	writeLock

Constructor Summary

Constructors

Constructor	Description
CommonNodeLabelsManager()

Method Summary

Modifier and Type	Method	Description
void	addLabelsToNode(java.util.Map<org.apache.hadoop.yarn.api.records.NodeId,java.util.Set<java.lang.String>> addedLabelsToNode)	add more labels to nodes.
void	addToCluserNodeLabels(java.util.Collection<org.apache.hadoop.yarn.api.records.NodeLabel> labels)
void	addToCluserNodeLabelsWithDefaultExclusivity(java.util.Set<java.lang.String> labels)	Add multiple node labels to repository.
protected void	checkAddLabelsToNode(java.util.Map<org.apache.hadoop.yarn.api.records.NodeId,java.util.Set<java.lang.String>> addedLabelsToNode)
protected void	checkRemoveFromClusterNodeLabels(java.util.Collection<java.lang.String> labelsToRemove)
protected void	checkRemoveLabelsFromNode(java.util.Map<org.apache.hadoop.yarn.api.records.NodeId,java.util.Set<java.lang.String>> removeLabelsFromNode)
protected void	checkReplaceLabelsOnNode(java.util.Map<org.apache.hadoop.yarn.api.records.NodeId,java.util.Set<java.lang.String>> replaceLabelsToNode)
protected void	createHostIfNonExisted(java.lang.String hostName)
protected void	createNodeIfNonExisted(org.apache.hadoop.yarn.api.records.NodeId nodeId)
java.util.Set<java.lang.String>	getClusterNodeLabelNames()	Get existing valid labels in repository
java.util.List<org.apache.hadoop.yarn.api.records.NodeLabel>	getClusterNodeLabels()
protected java.util.Set<java.lang.String>	getLabelsByNode(org.apache.hadoop.yarn.api.records.NodeId nodeId)
protected java.util.Set<java.lang.String>	getLabelsByNode(org.apache.hadoop.yarn.api.records.NodeId nodeId, java.util.Map<java.lang.String,CommonNodeLabelsManager.Host> map)
java.util.Set<org.apache.hadoop.yarn.api.records.NodeLabel>	getLabelsInfoByNode(org.apache.hadoop.yarn.api.records.NodeId nodeId)
java.util.Map<org.apache.hadoop.yarn.api.records.NodeLabel,java.util.Set<org.apache.hadoop.yarn.api.records.NodeId>>	getLabelsInfoToNodes()	Get mapping of labels to nodes for all the labels.
java.util.Map<org.apache.hadoop.yarn.api.records.NodeLabel,java.util.Set<org.apache.hadoop.yarn.api.records.NodeId>>	getLabelsInfoToNodes(java.util.Set<java.lang.String> labels)	Get mapping of labels info to nodes for specified set of labels.
java.util.Map<java.lang.String,java.util.Set<org.apache.hadoop.yarn.api.records.NodeId>>	getLabelsToNodes()	Get mapping of labels to nodes for all the labels.
java.util.Map<java.lang.String,java.util.Set<org.apache.hadoop.yarn.api.records.NodeId>>	getLabelsToNodes(java.util.Set<java.lang.String> labels)	Get mapping of labels to nodes for specified set of labels.
protected CommonNodeLabelsManager.Node	getNMInNodeSet(org.apache.hadoop.yarn.api.records.NodeId nodeId)
protected CommonNodeLabelsManager.Node	getNMInNodeSet(org.apache.hadoop.yarn.api.records.NodeId nodeId, java.util.Map<java.lang.String,CommonNodeLabelsManager.Host> map)
protected CommonNodeLabelsManager.Node	getNMInNodeSet(org.apache.hadoop.yarn.api.records.NodeId nodeId, java.util.Map<java.lang.String,CommonNodeLabelsManager.Host> map, boolean checkRunning)
java.util.Map<org.apache.hadoop.yarn.api.records.NodeId,java.util.Set<java.lang.String>>	getNodeLabels()	Get mapping of nodes to labels
java.util.Map<org.apache.hadoop.yarn.api.records.NodeId,java.util.Set<org.apache.hadoop.yarn.api.records.NodeLabel>>	getNodeLabelsInfo()	Get mapping of nodes to label info
java.util.Set<org.apache.hadoop.yarn.api.records.NodeId>	getNodesWithoutALabel()	Get nodes that have no labels.
protected void	handleStoreEvent(NodeLabelsStoreEvent event)
protected void	initDispatcher(org.apache.hadoop.conf.Configuration conf)
protected void	initNodeLabelStore(org.apache.hadoop.conf.Configuration conf)
protected void	internalRemoveFromClusterNodeLabels(java.util.Collection<java.lang.String> labelsToRemove)
protected void	internalUpdateLabelsOnNodes(java.util.Map<org.apache.hadoop.yarn.api.records.NodeId,java.util.Set<java.lang.String>> nodeToLabels, org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager.NodeLabelUpdateOperation op)
boolean	isCentralizedConfiguration()
boolean	isExclusiveNodeLabel(java.lang.String nodeLabel)
protected boolean	isInitNodeLabelStoreInProgress()
protected boolean	isNodeLabelExplicit(org.apache.hadoop.yarn.api.records.NodeId nodeId)
protected java.lang.String	normalizeLabel(java.lang.String label)
protected java.util.Map<org.apache.hadoop.yarn.api.records.NodeId,java.util.Set<java.lang.String>>	normalizeNodeIdToLabels(java.util.Map<org.apache.hadoop.yarn.api.records.NodeId,java.util.Set<java.lang.String>> nodeIdToLabels)
void	removeFromClusterNodeLabels(java.util.Collection<java.lang.String> labelsToRemove)	Remove multiple node labels from repository
void	removeLabelsFromNode(java.util.Map<org.apache.hadoop.yarn.api.records.NodeId,java.util.Set<java.lang.String>> removeLabelsFromNode)	remove labels from nodes, labels being removed most be contained by these nodes.
protected void	removeLabelsFromNodeInHost(org.apache.hadoop.yarn.api.records.NodeId node, java.util.Set<java.lang.String> labels)
protected void	removeNodeFromLabels(org.apache.hadoop.yarn.api.records.NodeId node, java.util.Set<java.lang.String> labels)
void	replaceLabelsOnNode(java.util.Map<org.apache.hadoop.yarn.api.records.NodeId,java.util.Set<java.lang.String>> replaceLabelsToNode)	replace labels to nodes
protected void	serviceInit(org.apache.hadoop.conf.Configuration conf)
protected void	serviceStart()
protected void	serviceStop()
void	setInitNodeLabelStoreInProgress(boolean initNodeLabelStoreInProgress)
protected void	startDispatcher()
protected void	stopDispatcher()

Methods inherited from class org.apache.hadoop.service.AbstractService

close, getBlockers, getConfig, getFailureCause, getFailureState, getLifecycleHistory, getName, getServiceState, getStartTime, init, isInState, noteFailure, putBlocker, registerGlobalListener, registerServiceListener, removeBlocker, setConfig, start, stop, toString, unregisterGlobalListener, unregisterServiceListener, waitForServiceToStop

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, wait, wait, wait

Field Detail

LOG

protected static final org.slf4j.Logger LOG

EMPTY_STRING_SET

public static final java.util.Set<java.lang.String> EMPTY_STRING_SET

EMPTY_NODELABEL_SET

public static final java.util.Set<org.apache.hadoop.yarn.api.records.NodeLabel> EMPTY_NODELABEL_SET

ANY

public static final java.lang.String ANY

See Also:

Constant Field Values

ACCESS_ANY_LABEL_SET

public static final java.util.Set<java.lang.String> ACCESS_ANY_LABEL_SET

WILDCARD_PORT

public static final int WILDCARD_PORT

See Also:

Constant Field Values

NODE_LABELS_NOT_ENABLED_ERR

@VisibleForTesting
public static final java.lang.String NODE_LABELS_NOT_ENABLED_ERR

Error messages

See Also:

Constant Field Values

NO_LABEL

public static final java.lang.String NO_LABEL

If a user doesn't specify label of a queue or node, it belongs DEFAULT_LABEL

See Also:

Constant Field Values

dispatcher

protected Dispatcher dispatcher

labelCollections

protected java.util.concurrent.ConcurrentMap<java.lang.String,RMNodeLabel> labelCollections

nodeCollections

protected java.util.concurrent.ConcurrentMap<java.lang.String,CommonNodeLabelsManager.Host> nodeCollections

noNodeLabel

protected RMNodeLabel noNodeLabel

readLock

protected final java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock readLock

writeLock

protected final java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock writeLock

store

protected NodeLabelsStore store

Constructor Detail

CommonNodeLabelsManager

public CommonNodeLabelsManager()

Method Detail

handleStoreEvent

protected void handleStoreEvent(NodeLabelsStoreEvent event)

initDispatcher

protected void initDispatcher(org.apache.hadoop.conf.Configuration conf)

serviceInit

protected void serviceInit(org.apache.hadoop.conf.Configuration conf)
                    throws java.lang.Exception

Overrides:

serviceInit in class org.apache.hadoop.service.AbstractService

Throws:

java.lang.Exception

isInitNodeLabelStoreInProgress

protected boolean isInitNodeLabelStoreInProgress()

Returns:

the isStartup

isCentralizedConfiguration

public boolean isCentralizedConfiguration()

Returns:

true if node label configuration type is not distributed.

initNodeLabelStore

protected void initNodeLabelStore(org.apache.hadoop.conf.Configuration conf)
                           throws java.lang.Exception

Throws:

java.lang.Exception

startDispatcher

protected void startDispatcher()

serviceStart

protected void serviceStart()
                     throws java.lang.Exception

Overrides:

serviceStart in class org.apache.hadoop.service.AbstractService

Throws:

java.lang.Exception

stopDispatcher

protected void stopDispatcher()

serviceStop

protected void serviceStop()
                    throws java.lang.Exception

Overrides:

serviceStop in class org.apache.hadoop.service.AbstractService

Throws:

java.lang.Exception

addToCluserNodeLabels

public void addToCluserNodeLabels(java.util.Collection<org.apache.hadoop.yarn.api.records.NodeLabel> labels)
                           throws java.io.IOException

Throws:

java.io.IOException

addToCluserNodeLabelsWithDefaultExclusivity

@VisibleForTesting
public void addToCluserNodeLabelsWithDefaultExclusivity(java.util.Set<java.lang.String> labels)
                                                 throws java.io.IOException

Add multiple node labels to repository.

Parameters:

labels - new node labels added

Throws:

java.io.IOException - io error occur.

checkAddLabelsToNode

protected void checkAddLabelsToNode(java.util.Map<org.apache.hadoop.yarn.api.records.NodeId,java.util.Set<java.lang.String>> addedLabelsToNode)
                             throws java.io.IOException

Throws:

java.io.IOException

addLabelsToNode

public void addLabelsToNode(java.util.Map<org.apache.hadoop.yarn.api.records.NodeId,java.util.Set<java.lang.String>> addedLabelsToNode)
                     throws java.io.IOException

add more labels to nodes.

Parameters:

addedLabelsToNode - node -> labels map

Throws:

java.io.IOException - io error occur.

checkRemoveFromClusterNodeLabels

protected void checkRemoveFromClusterNodeLabels(java.util.Collection<java.lang.String> labelsToRemove)
                                         throws java.io.IOException

Throws:

java.io.IOException

internalRemoveFromClusterNodeLabels

protected void internalRemoveFromClusterNodeLabels(java.util.Collection<java.lang.String> labelsToRemove)

removeFromClusterNodeLabels

public void removeFromClusterNodeLabels(java.util.Collection<java.lang.String> labelsToRemove)
                                 throws java.io.IOException

Remove multiple node labels from repository

Parameters:

labelsToRemove - node labels to remove

Throws:

java.io.IOException - io error occur.

checkRemoveLabelsFromNode

protected void checkRemoveLabelsFromNode(java.util.Map<org.apache.hadoop.yarn.api.records.NodeId,java.util.Set<java.lang.String>> removeLabelsFromNode)
                                  throws java.io.IOException

Throws:

java.io.IOException

removeNodeFromLabels

protected void removeNodeFromLabels(org.apache.hadoop.yarn.api.records.NodeId node,
                                    java.util.Set<java.lang.String> labels)

removeLabelsFromNodeInHost

protected void removeLabelsFromNodeInHost(org.apache.hadoop.yarn.api.records.NodeId node,
                                          java.util.Set<java.lang.String> labels)
                                   throws java.io.IOException

Throws:

java.io.IOException

isNodeLabelExplicit

protected boolean isNodeLabelExplicit(org.apache.hadoop.yarn.api.records.NodeId nodeId)

internalUpdateLabelsOnNodes

protected void internalUpdateLabelsOnNodes(java.util.Map<org.apache.hadoop.yarn.api.records.NodeId,java.util.Set<java.lang.String>> nodeToLabels,
                                           org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager.NodeLabelUpdateOperation op)
                                    throws java.io.IOException

Throws:

java.io.IOException

removeLabelsFromNode

public void removeLabelsFromNode(java.util.Map<org.apache.hadoop.yarn.api.records.NodeId,java.util.Set<java.lang.String>> removeLabelsFromNode)
                          throws java.io.IOException

remove labels from nodes, labels being removed most be contained by these nodes.

Parameters:

removeLabelsFromNode - node -> labels map

Throws:

java.io.IOException - io error occur.

checkReplaceLabelsOnNode

protected void checkReplaceLabelsOnNode(java.util.Map<org.apache.hadoop.yarn.api.records.NodeId,java.util.Set<java.lang.String>> replaceLabelsToNode)
                                 throws java.io.IOException

Throws:

java.io.IOException

replaceLabelsOnNode

public void replaceLabelsOnNode(java.util.Map<org.apache.hadoop.yarn.api.records.NodeId,java.util.Set<java.lang.String>> replaceLabelsToNode)
                         throws java.io.IOException

replace labels to nodes

Parameters:

replaceLabelsToNode - node -> labels map

Throws:

java.io.IOException - io error occur.

getNodeLabels

public java.util.Map<org.apache.hadoop.yarn.api.records.NodeId,java.util.Set<java.lang.String>> getNodeLabels()

Get mapping of nodes to labels

Returns:

nodes to labels map

getNodeLabelsInfo

public java.util.Map<org.apache.hadoop.yarn.api.records.NodeId,java.util.Set<org.apache.hadoop.yarn.api.records.NodeLabel>> getNodeLabelsInfo()

Get mapping of nodes to label info

Returns:

nodes to labels map

getNodesWithoutALabel

public java.util.Set<org.apache.hadoop.yarn.api.records.NodeId> getNodesWithoutALabel()

Get nodes that have no labels.

Returns:

set of nodes with no labels

getLabelsToNodes

public java.util.Map<java.lang.String,java.util.Set<org.apache.hadoop.yarn.api.records.NodeId>> getLabelsToNodes()

Get mapping of labels to nodes for all the labels.

Returns:

labels to nodes map

getLabelsToNodes

public java.util.Map<java.lang.String,java.util.Set<org.apache.hadoop.yarn.api.records.NodeId>> getLabelsToNodes(java.util.Set<java.lang.String> labels)

Get mapping of labels to nodes for specified set of labels.

Parameters:

labels - set of labels for which labels to nodes mapping will be returned.

Returns:

labels to nodes map

getLabelsInfoToNodes

public java.util.Map<org.apache.hadoop.yarn.api.records.NodeLabel,java.util.Set<org.apache.hadoop.yarn.api.records.NodeId>> getLabelsInfoToNodes()

Get mapping of labels to nodes for all the labels.

Returns:

labels to nodes map

getLabelsInfoToNodes

public java.util.Map<org.apache.hadoop.yarn.api.records.NodeLabel,java.util.Set<org.apache.hadoop.yarn.api.records.NodeId>> getLabelsInfoToNodes(java.util.Set<java.lang.String> labels)

Get mapping of labels info to nodes for specified set of labels.

Parameters:

labels - set of nodelabels for which labels to nodes mapping will be returned.

Returns:

labels to nodes map

getClusterNodeLabelNames

public java.util.Set<java.lang.String> getClusterNodeLabelNames()

Get existing valid labels in repository

Returns:

existing valid labels in repository

getClusterNodeLabels

public java.util.List<org.apache.hadoop.yarn.api.records.NodeLabel> getClusterNodeLabels()

isExclusiveNodeLabel

public boolean isExclusiveNodeLabel(java.lang.String nodeLabel)
                             throws java.io.IOException

Throws:

java.io.IOException

normalizeLabel

protected java.lang.String normalizeLabel(java.lang.String label)

getNMInNodeSet

protected CommonNodeLabelsManager.Node getNMInNodeSet(org.apache.hadoop.yarn.api.records.NodeId nodeId)

getNMInNodeSet

protected CommonNodeLabelsManager.Node getNMInNodeSet(org.apache.hadoop.yarn.api.records.NodeId nodeId,
                                                      java.util.Map<java.lang.String,CommonNodeLabelsManager.Host> map)

getNMInNodeSet

protected CommonNodeLabelsManager.Node getNMInNodeSet(org.apache.hadoop.yarn.api.records.NodeId nodeId,
                                                      java.util.Map<java.lang.String,CommonNodeLabelsManager.Host> map,
                                                      boolean checkRunning)

getLabelsByNode

protected java.util.Set<java.lang.String> getLabelsByNode(org.apache.hadoop.yarn.api.records.NodeId nodeId)

getLabelsByNode

protected java.util.Set<java.lang.String> getLabelsByNode(org.apache.hadoop.yarn.api.records.NodeId nodeId,
                                                          java.util.Map<java.lang.String,CommonNodeLabelsManager.Host> map)

getLabelsInfoByNode

public java.util.Set<org.apache.hadoop.yarn.api.records.NodeLabel> getLabelsInfoByNode(org.apache.hadoop.yarn.api.records.NodeId nodeId)

createNodeIfNonExisted

protected void createNodeIfNonExisted(org.apache.hadoop.yarn.api.records.NodeId nodeId)
                               throws java.io.IOException

Throws:

java.io.IOException

createHostIfNonExisted

protected void createHostIfNonExisted(java.lang.String hostName)

normalizeNodeIdToLabels

protected java.util.Map<org.apache.hadoop.yarn.api.records.NodeId,java.util.Set<java.lang.String>> normalizeNodeIdToLabels(java.util.Map<org.apache.hadoop.yarn.api.records.NodeId,java.util.Set<java.lang.String>> nodeIdToLabels)

setInitNodeLabelStoreInProgress

public void setInitNodeLabelStoreInProgress(boolean initNodeLabelStoreInProgress)