Package org.apache.hadoop.hdfs.server.blockmanagement

Class DatanodeManager

java.lang.Object

org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager

@Private
@Evolving
public class DatanodeManager
extends java.lang.Object

Manage datanodes, include decommission and other activities.

Method Summary

Modifier and Type	Method	Description
void	addSlowPeers(java.lang.String dnUuid)
void	clearPendingCachingCommands()
void	clearPendingQueues()	Clear any actions that are queued up to be sent to the DNs on their next heartbeats.
void	fetchDatanodes(java.util.List<DatanodeDescriptor> live, java.util.List<DatanodeDescriptor> dead, boolean removeDecommissionNode)	Fetch live and dead datanodes.
java.util.List<DatanodeDescriptor>	getAllSlowDataNodes()
int	getBlockInvalidateLimit()
long	getBlocksPerPostponedMisreplicatedBlocksRescan()
DatanodeDescriptor	getDatanode(java.lang.String datanodeUuid)	Get a datanode descriptor given corresponding DatanodeUUID
DatanodeDescriptor	getDatanode(org.apache.hadoop.hdfs.protocol.DatanodeID nodeID)	Get data node by datanode ID.
DatanodeAdminManager	getDatanodeAdminManager()
DatanodeDescriptor	getDatanodeByHost(java.lang.String host)
DatanodeDescriptor	getDatanodeByXferAddr(java.lang.String host, int xferPort)
java.util.List<DatanodeDescriptor>	getDatanodeListForReport(org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType type)	For generating datanode reports
java.util.Map<java.lang.String,DatanodeDescriptor>	getDatanodeMap()
java.util.Set<DatanodeDescriptor>	getDatanodes()
java.util.HashMap<java.lang.String,java.lang.Integer>	getDatanodesSoftwareVersions()
DatanodeStatistics	getDatanodeStatistics()
DatanodeStorageInfo[]	getDatanodeStorageInfos(org.apache.hadoop.hdfs.protocol.DatanodeID[] datanodeID, java.lang.String[] storageIDs, java.lang.String format, java.lang.Object... args)
org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport[]	getDatanodeStorageReport(org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType type)	Generates datanode reports for the given report type.
java.util.List<DatanodeDescriptor>	getDecommissioningNodes()
boolean	getEnableAvoidSlowDataNodesForRead()
java.util.List<DatanodeDescriptor>	getEnteringMaintenanceNodes()
FSClusterStats	getFSClusterStats()
long	getHeartbeatInterval()
long	getHeartbeatRecheckInterval()
org.apache.hadoop.hdfs.server.blockmanagement.Host2NodesMap	getHost2DatanodeMap()
HostConfigManager	getHostConfigManager()
int	getMaxSlowpeerCollectNodes()
org.apache.hadoop.net.NetworkTopology	getNetworkTopology()
int	getNumDeadDataNodes()
int	getNumLiveDataNodes()
int	getNumOfDataNodes()
int	getNumStaleNodes()
int	getNumStaleStorages()	Get the number of content stale storages.
java.lang.String	getSlowDisksReport()	Retrieve information about slow disks as a JSON.
SlowDiskTracker	getSlowDiskTracker()	Use only for testing.
static java.util.Set<java.lang.String>	getSlowNodesUuidSet()	Returns all tracking slow datanodes uuids.
long	getSlowPeerCollectionInterval()
java.lang.String	getSlowPeersReport()	Retrieve information about slow peers as a JSON.
java.util.Set<java.lang.String>	getSlowPeersUuidSet()	Returns all tracking slow peers.
SlowPeerTracker	getSlowPeerTracker()	Use only for testing.
DatanodeCommand[]	handleHeartbeat(DatanodeRegistration nodeReg, org.apache.hadoop.hdfs.server.protocol.StorageReport[] reports, java.lang.String blockPoolId, long cacheCapacity, long cacheUsed, int xceiverCount, int xmitsInProgress, int failedVolumes, VolumeFailureSummary volumeFailureSummary, org.apache.hadoop.hdfs.server.protocol.SlowPeerReports slowPeers, org.apache.hadoop.hdfs.server.protocol.SlowDiskReports slowDisks)	Handle heartbeat from datanodes.
void	handleLifeline(DatanodeRegistration nodeReg, org.apache.hadoop.hdfs.server.protocol.StorageReport[] reports, long cacheCapacity, long cacheUsed, int xceiverCount, int failedVolumes, VolumeFailureSummary volumeFailureSummary)	Handles a lifeline message sent by a DataNode.
void	initSlowPeerTracker(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.util.Timer timer, boolean dataNodePeerStatsEnabled)	Determines whether slow peer tracker should be enabled.
boolean	isSlowPeerCollectorInitialized()
void	markAllDatanodesStale()
void	refreshNodes(org.apache.hadoop.conf.Configuration conf)	Rereads conf to get hosts and exclude list file names.
void	registerDatanode(DatanodeRegistration nodeReg)	Register the given datanode with the namenode.
void	removeDatanode(org.apache.hadoop.hdfs.protocol.DatanodeID node)	Remove a datanode
void	resetLastCachingDirectiveSentTime()	Reset the lastCachingDirectiveSentTimeMs field of all the DataNodes we know about.
java.util.List<java.lang.String>	resolveNetworkLocation(java.util.List<java.lang.String> names)	Resolve network locations for specified hosts
void	restartSlowPeerCollector(long interval)
void	setAvoidSlowDataNodesForReadEnabled(boolean enable)
void	setBalancerBandwidth(long bandwidth)	Tell all datanodes to use a new, non-persistent bandwidth value for dfs.datanode.balance.bandwidthPerSec.
void	setBlockInvalidateLimit(int configuredBlockInvalidateLimit)
void	setHeartbeatExpireInterval(long expiryMs)
void	setHeartbeatInterval(long intervalSeconds)
void	setHeartbeatRecheckInterval(int recheckInterval)
void	setMaxSlowpeerCollectNodes(int maxNodes)
void	setMaxSlowPeersToReport(int maxSlowPeersToReport)
void	setShouldSendCachingCommands(boolean shouldSendCachingCommands)
boolean	shouldAvoidStaleDataNodesForWrite()	Whether stale datanodes should be avoided as targets on the write path.
void	sortLocatedBlocks(java.lang.String targetHost, java.util.List<org.apache.hadoop.hdfs.protocol.LocatedBlock> locatedBlocks)	Sort the non-striped located blocks by the distance to the target host.
java.lang.String	toString()

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, wait, wait, wait

Method Detail

initSlowPeerTracker

public void initSlowPeerTracker(org.apache.hadoop.conf.Configuration conf,
                                org.apache.hadoop.util.Timer timer,
                                boolean dataNodePeerStatsEnabled)

Determines whether slow peer tracker should be enabled. If dataNodePeerStatsEnabledVal is true, slow peer tracker is initialized.

Parameters:

conf - The configuration to use while initializing slowPeerTracker.

timer - Timer object for slowPeerTracker.

dataNodePeerStatsEnabled - To determine whether slow peer tracking should be enabled.

restartSlowPeerCollector

public void restartSlowPeerCollector(long interval)

getNetworkTopology

public org.apache.hadoop.net.NetworkTopology getNetworkTopology()

Returns:

the network topology.

getDatanodeAdminManager

@VisibleForTesting
public DatanodeAdminManager getDatanodeAdminManager()

getHostConfigManager

public HostConfigManager getHostConfigManager()

setHeartbeatExpireInterval

@VisibleForTesting
public void setHeartbeatExpireInterval(long expiryMs)

getFSClusterStats

@VisibleForTesting
public FSClusterStats getFSClusterStats()

getBlockInvalidateLimit

@VisibleForTesting
public int getBlockInvalidateLimit()

getDatanodeStatistics

public DatanodeStatistics getDatanodeStatistics()

Returns:

the datanode statistics.

setAvoidSlowDataNodesForReadEnabled

public void setAvoidSlowDataNodesForReadEnabled(boolean enable)

getEnableAvoidSlowDataNodesForRead

@VisibleForTesting
public boolean getEnableAvoidSlowDataNodesForRead()

setMaxSlowpeerCollectNodes

public void setMaxSlowpeerCollectNodes(int maxNodes)

getMaxSlowpeerCollectNodes

@VisibleForTesting
public int getMaxSlowpeerCollectNodes()

sortLocatedBlocks

public void sortLocatedBlocks(java.lang.String targetHost,
                              java.util.List<org.apache.hadoop.hdfs.protocol.LocatedBlock> locatedBlocks)

Sort the non-striped located blocks by the distance to the target host. For striped blocks, it will only move decommissioned/decommissioning/stale/slow nodes to the bottom. For example, assume we have storage list: d0, d1, d2, d3, d4, d5, d6, d7, d8, d9 mapping to block indices: 0, 1, 2, 3, 4, 5, 6, 7, 8, 2 Here the internal block b2 is duplicated, locating in d2 and d9. If d2 is a decommissioning node then should switch d2 and d9 in the storage list. After sorting locations, will update corresponding block indices and block tokens.

getDatanodeByHost

public DatanodeDescriptor getDatanodeByHost(java.lang.String host)

Returns:

the datanode descriptor for the host.

getDatanodeByXferAddr

public DatanodeDescriptor getDatanodeByXferAddr(java.lang.String host,
                                                int xferPort)

Returns:

the datanode descriptor for the host.

getDatanodes

public java.util.Set<DatanodeDescriptor> getDatanodes()

Returns:

the datanode descriptors for all nodes.

getHost2DatanodeMap

public org.apache.hadoop.hdfs.server.blockmanagement.Host2NodesMap getHost2DatanodeMap()

Returns:

the Host2NodesMap

getDatanode

public DatanodeDescriptor getDatanode(java.lang.String datanodeUuid)

Get a datanode descriptor given corresponding DatanodeUUID

getDatanode

public DatanodeDescriptor getDatanode(org.apache.hadoop.hdfs.protocol.DatanodeID nodeID)
                               throws UnregisteredNodeException

Get data node by datanode ID.

Parameters:

nodeID - datanode ID

Returns:

DatanodeDescriptor or null if the node is not found.

Throws:

UnregisteredNodeException

getDatanodeStorageInfos

public DatanodeStorageInfo[] getDatanodeStorageInfos(org.apache.hadoop.hdfs.protocol.DatanodeID[] datanodeID,
                                                     java.lang.String[] storageIDs,
                                                     java.lang.String format,
                                                     java.lang.Object... args)
                                              throws UnregisteredNodeException

Throws:

UnregisteredNodeException

removeDatanode

public void removeDatanode(org.apache.hadoop.hdfs.protocol.DatanodeID node)
                    throws UnregisteredNodeException

Remove a datanode

Throws:

UnregisteredNodeException

getDatanodesSoftwareVersions

public java.util.HashMap<java.lang.String,java.lang.Integer> getDatanodesSoftwareVersions()

resolveNetworkLocation

public java.util.List<java.lang.String> resolveNetworkLocation(java.util.List<java.lang.String> names)

Resolve network locations for specified hosts

Returns:

Network locations if available, Else returns null

registerDatanode

public void registerDatanode(DatanodeRegistration nodeReg)
                      throws DisallowedDatanodeException,
                             UnresolvedTopologyException

Register the given datanode with the namenode. NB: the given registration is mutated and given back to the datanode.

Parameters:

nodeReg - the datanode registration

Throws:

DisallowedDatanodeException - if the registration request is denied because the datanode does not match includes/excludes

UnresolvedTopologyException - if the registration request is denied because resolving datanode network location fails.

refreshNodes

public void refreshNodes(org.apache.hadoop.conf.Configuration conf)
                  throws java.io.IOException

Rereads conf to get hosts and exclude list file names. Rereads the files to update the hosts and exclude lists. It checks if any of the hosts have changed states:

Throws:

java.io.IOException

getNumLiveDataNodes

public int getNumLiveDataNodes()

Returns:

the number of live datanodes.

getNumDeadDataNodes

public int getNumDeadDataNodes()

Returns:

the number of dead datanodes.

getNumOfDataNodes

public int getNumOfDataNodes()

Returns:

the number of datanodes.

getDecommissioningNodes

public java.util.List<DatanodeDescriptor> getDecommissioningNodes()

Returns:

list of datanodes where decommissioning is in progress.

getEnteringMaintenanceNodes

public java.util.List<DatanodeDescriptor> getEnteringMaintenanceNodes()

Returns:

list of datanodes that are entering maintenance.

shouldAvoidStaleDataNodesForWrite

public boolean shouldAvoidStaleDataNodesForWrite()

Whether stale datanodes should be avoided as targets on the write path. The result of this function may change if the number of stale datanodes eclipses a configurable threshold.

Returns:

whether stale datanodes should be avoided on the write path

getBlocksPerPostponedMisreplicatedBlocksRescan

public long getBlocksPerPostponedMisreplicatedBlocksRescan()

getHeartbeatInterval

public long getHeartbeatInterval()

getHeartbeatRecheckInterval

public long getHeartbeatRecheckInterval()

getNumStaleNodes

public int getNumStaleNodes()

Returns:

Return the current number of stale DataNodes (detected by HeartbeatManager).

getNumStaleStorages

public int getNumStaleStorages()

Get the number of content stale storages.

fetchDatanodes

public void fetchDatanodes(java.util.List<DatanodeDescriptor> live,
                           java.util.List<DatanodeDescriptor> dead,
                           boolean removeDecommissionNode)

Fetch live and dead datanodes.

getDatanodeListForReport

public java.util.List<DatanodeDescriptor> getDatanodeListForReport(org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType type)

For generating datanode reports

getAllSlowDataNodes

public java.util.List<DatanodeDescriptor> getAllSlowDataNodes()

handleHeartbeat

public DatanodeCommand[] handleHeartbeat(DatanodeRegistration nodeReg,
                                         org.apache.hadoop.hdfs.server.protocol.StorageReport[] reports,
                                         java.lang.String blockPoolId,
                                         long cacheCapacity,
                                         long cacheUsed,
                                         int xceiverCount,
                                         int xmitsInProgress,
                                         int failedVolumes,
                                         VolumeFailureSummary volumeFailureSummary,
                                         @Nonnull
                                         org.apache.hadoop.hdfs.server.protocol.SlowPeerReports slowPeers,
                                         @Nonnull
                                         org.apache.hadoop.hdfs.server.protocol.SlowDiskReports slowDisks)
                                  throws java.io.IOException

Handle heartbeat from datanodes.

Throws:

java.io.IOException

handleLifeline

public void handleLifeline(DatanodeRegistration nodeReg,
                           org.apache.hadoop.hdfs.server.protocol.StorageReport[] reports,
                           long cacheCapacity,
                           long cacheUsed,
                           int xceiverCount,
                           int failedVolumes,
                           VolumeFailureSummary volumeFailureSummary)
                    throws java.io.IOException

Handles a lifeline message sent by a DataNode.

Parameters:

nodeReg - registration info for DataNode sending the lifeline

reports - storage reports from DataNode

cacheCapacity - cache capacity at DataNode

cacheUsed - cache used at DataNode

xceiverCount - estimated count of transfer threads running at DataNode

failedVolumes - count of failed volumes at DataNode

volumeFailureSummary - info on failed volumes at DataNode

Throws:

java.io.IOException - if there is an error

setBalancerBandwidth

public void setBalancerBandwidth(long bandwidth)
                          throws java.io.IOException

Tell all datanodes to use a new, non-persistent bandwidth value for dfs.datanode.balance.bandwidthPerSec. A system administrator can tune the balancer bandwidth parameter (dfs.datanode.balance.bandwidthPerSec) dynamically by calling "dfsadmin -setBalanacerBandwidth newbandwidth", at which point the following 'bandwidth' variable gets updated with the new value for each node. Once the heartbeat command is issued to update the value on the specified datanode, this value will be set back to 0.

Parameters:

bandwidth - Blanacer bandwidth in bytes per second for all datanodes.

Throws:

java.io.IOException

markAllDatanodesStale

public void markAllDatanodesStale()

clearPendingQueues

public void clearPendingQueues()

Clear any actions that are queued up to be sent to the DNs on their next heartbeats. This includes block invalidations, recoveries, and replication requests.

resetLastCachingDirectiveSentTime

public void resetLastCachingDirectiveSentTime()

Reset the lastCachingDirectiveSentTimeMs field of all the DataNodes we know about.

toString

public java.lang.String toString()

Overrides:

toString in class java.lang.Object

clearPendingCachingCommands

public void clearPendingCachingCommands()

setShouldSendCachingCommands

public void setShouldSendCachingCommands(boolean shouldSendCachingCommands)

setHeartbeatInterval

public void setHeartbeatInterval(long intervalSeconds)

setHeartbeatRecheckInterval

public void setHeartbeatRecheckInterval(int recheckInterval)

setBlockInvalidateLimit

public void setBlockInvalidateLimit(int configuredBlockInvalidateLimit)

getSlowPeersReport

public java.lang.String getSlowPeersReport()

Retrieve information about slow peers as a JSON. Returns null if we are not tracking slow peers.

Returns:

getSlowPeersUuidSet

public java.util.Set<java.lang.String> getSlowPeersUuidSet()

Returns all tracking slow peers.

Returns:

getSlowNodesUuidSet

public static java.util.Set<java.lang.String> getSlowNodesUuidSet()

Returns all tracking slow datanodes uuids.

Returns:

getSlowPeerTracker

@VisibleForTesting
public SlowPeerTracker getSlowPeerTracker()

Use only for testing.

getSlowDiskTracker

@VisibleForTesting
public SlowDiskTracker getSlowDiskTracker()

Use only for testing.

addSlowPeers

@VisibleForTesting
public void addSlowPeers(java.lang.String dnUuid)

getSlowDisksReport

public java.lang.String getSlowDisksReport()

Retrieve information about slow disks as a JSON. Returns null if we are not tracking slow disks.

Returns:

getDatanodeStorageReport

public org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport[] getDatanodeStorageReport(org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType type)

Generates datanode reports for the given report type.

Parameters:

type - type of the datanode report

Returns:

array of DatanodeStorageReports

getDatanodeMap

@VisibleForTesting
public java.util.Map<java.lang.String,DatanodeDescriptor> getDatanodeMap()

setMaxSlowPeersToReport

public void setMaxSlowPeersToReport(int maxSlowPeersToReport)

isSlowPeerCollectorInitialized

@VisibleForTesting
public boolean isSlowPeerCollectorInitialized()

getSlowPeerCollectionInterval

@VisibleForTesting
public long getSlowPeerCollectionInterval()