Package org.apache.hadoop.hdfs

Class DFSUtilClient

java.lang.Object

org.apache.hadoop.hdfs.DFSUtilClient

@Private
public class DFSUtilClient
extends java.lang.Object

Nested Class Summary

Nested Classes

Modifier and Type	Class	Description
static class	DFSUtilClient.CorruptedBlocks	A utility class as a container to put corrupted blocks, shared by client and datanode.
static interface	DFSUtilClient.SnapshotDiffReportFunction
static interface	DFSUtilClient.SnapshotDiffReportListingFunction

Field Summary

Fields

Modifier and Type	Field	Description
static byte[]	EMPTY_BYTES

Constructor Summary

Constructors

Constructor	Description
DFSUtilClient()

Method Summary

Modifier and Type	Method	Description
static byte[]	byteArray2bytes(byte[][] pathComponents)	Given a list of path components returns a byte array
static java.lang.String	byteArray2String(byte[][] pathComponents)	Given a list of path components returns a string.
static byte[][]	bytes2byteArray(byte[] bytes)	Converts a byte array to array of arrays of bytes on byte separator.
static byte[][]	bytes2byteArray(byte[] bytes, int len, byte separator)	Splits first len bytes in bytes to array of arrays of bytes on byte separator.
static java.lang.String	bytes2String(byte[] bytes)	Converts a byte array to a string using UTF8 encoding.
static int	compareBytes(byte[] left, byte[] right)	Compare two byte arrays by lexicographical order.
static IOStreamPair	connectToDN(DatanodeInfo dn, int timeout, org.apache.hadoop.conf.Configuration conf, SaslDataTransferClient saslClient, javax.net.SocketFactory socketFactory, boolean connectToDnViaHostname, DataEncryptionKeyFactory dekFactory, org.apache.hadoop.security.token.Token<BlockTokenIdentifier> blockToken)	Connect to the given datanode's datantrasfer port, and return the resulting IOStreamPair.
static ClientDatanodeProtocol	createClientDatanodeProtocolProxy(java.net.InetSocketAddress addr, org.apache.hadoop.security.UserGroupInformation ticket, org.apache.hadoop.conf.Configuration conf, javax.net.SocketFactory factory)	Create a ClientDatanodeProtocol proxy
static ClientDatanodeProtocol	createClientDatanodeProtocolProxy(DatanodeID datanodeid, org.apache.hadoop.conf.Configuration conf, int socketTimeout, boolean connectToDnViaHostname)	Create ClientDatanodeProtocol proxy using kerberos ticket
static ClientDatanodeProtocol	createClientDatanodeProtocolProxy(DatanodeID datanodeid, org.apache.hadoop.conf.Configuration conf, int socketTimeout, boolean connectToDnViaHostname, LocatedBlock locatedBlock)	Create a ClientDatanodeProtocol proxy
static ReconfigurationProtocol	createReconfigurationProtocolProxy(java.net.InetSocketAddress addr, org.apache.hadoop.security.UserGroupInformation ticket, org.apache.hadoop.conf.Configuration conf, javax.net.SocketFactory factory)
static java.lang.String	dateToIso8601String(java.util.Date date)	Converts a Date into an ISO-8601 formatted datetime string.
static java.lang.String	durationToString(long durationMs)	Converts a time duration in milliseconds into DDD:HH:MM:SS format.
static java.util.Map<java.lang.String,java.util.Map<java.lang.String,java.net.InetSocketAddress>>	getAddresses(org.apache.hadoop.conf.Configuration conf, java.lang.String defaultAddress, java.lang.String... keys)	Returns the configured address for all NameNodes in the cluster.
static java.util.Map<java.lang.String,java.net.InetSocketAddress>	getAddressesForNameserviceId(org.apache.hadoop.conf.Configuration conf, java.lang.String nsId, java.lang.String defaultValue, java.lang.String... keys)
static java.lang.String	getConfValue(java.lang.String defaultValue, java.lang.String keySuffix, org.apache.hadoop.conf.Configuration conf, java.lang.String... keys)	Given a list of keys in the order of preference, returns a value for the key in the given order from the configuration.
static java.lang.String	getEZTrashRoot(EncryptionZone ez, org.apache.hadoop.security.UserGroupInformation ugi)	Returns trash root in encryption zone.
static java.util.Map<java.lang.String,java.util.Map<java.lang.String,java.net.InetSocketAddress>>	getHaNnRpcAddresses(org.apache.hadoop.conf.Configuration conf)	Returns list of InetSocketAddress corresponding to HA NN RPC addresses from the configuration.
static java.util.Map<java.lang.String,java.util.Map<java.lang.String,java.net.InetSocketAddress>>	getHaNnWebHdfsAddresses(org.apache.hadoop.conf.Configuration conf, java.lang.String scheme)	Returns list of InetSocketAddress corresponding to HA NN HTTP addresses from the configuration.
static java.lang.String	getHomeDirectory(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.security.UserGroupInformation ugi)	Returns current user home directory under a home directory prefix.
static int	getIoFileBufferSize(org.apache.hadoop.conf.Configuration conf)
static java.util.Collection<java.lang.String>	getNameNodeIds(org.apache.hadoop.conf.Configuration conf, java.lang.String nsId)	Namenode HighAvailability related configuration.
static java.util.Collection<java.lang.String>	getNameServiceIds(org.apache.hadoop.conf.Configuration conf)	Returns collection of nameservice Ids from the configuration.
static java.net.InetSocketAddress	getNNAddress(java.lang.String address)
static java.net.InetSocketAddress	getNNAddress(java.net.URI filesystemURI)
static java.net.InetSocketAddress	getNNAddress(org.apache.hadoop.conf.Configuration conf)
static java.net.InetSocketAddress	getNNAddressCheckLogical(org.apache.hadoop.conf.Configuration conf, java.net.URI filesystemURI)	Get the NN address from the URI.
static java.net.URI	getNNUri(java.net.InetSocketAddress namenode)
static float	getPercentRemaining(long remaining, long capacity)	Return remaining as percentage of capacity
static float	getPercentUsed(long used, long capacity)	Return used as percentage of capacity
static java.util.Map<java.lang.String,java.net.InetSocketAddress>	getResolvedAddressesForNnId(org.apache.hadoop.conf.Configuration conf, java.lang.String nsId, java.lang.String nnId, org.apache.hadoop.net.DomainNameResolver dnr, java.lang.String defaultValue, java.lang.String... keys)
static int	getSmallBufferSize(org.apache.hadoop.conf.Configuration conf)
static SnapshotDiffReport	getSnapshotDiffReport(java.lang.String snapshotDir, java.lang.String fromSnapshot, java.lang.String toSnapshot, DFSUtilClient.SnapshotDiffReportFunction withoutListing, DFSUtilClient.SnapshotDiffReportListingFunction withListing)
static java.lang.String	getSnapshotTrashRoot(java.lang.String ssRoot, org.apache.hadoop.security.UserGroupInformation ugi)	Returns trash root in a snapshottable directory.
static java.util.concurrent.ThreadPoolExecutor	getThreadPoolExecutor(int corePoolSize, int maxPoolSize, long keepAliveTimeSecs, java.lang.String threadNamePrefix, boolean runRejectedExec)	Utility to create a ThreadPoolExecutor.
static java.util.concurrent.ThreadPoolExecutor	getThreadPoolExecutor(int corePoolSize, int maxPoolSize, long keepAliveTimeSecs, java.util.concurrent.BlockingQueue<java.lang.Runnable> queue, java.lang.String threadNamePrefix, boolean runRejectedExec)	Utility to create a ThreadPoolExecutor.
static java.lang.String	getTrashRoot(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.security.UserGroupInformation ugi)	Returns trash root in non-encryption zone.
static boolean	isHDFSEncryptionEnabled(org.apache.hadoop.conf.Configuration conf)	Probe for HDFS Encryption being enabled; this uses the value of the option CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH , returning true if that property contains a non-empty, non-whitespace string.
static boolean	isLocalAddress(java.net.InetSocketAddress targetAddr)
static boolean	isValidName(java.lang.String src)	Whether the pathname is valid.
static boolean	isValidSnapshotName(java.lang.String snapshotName)	Returns true if the name of snapshot is vlaid.
static org.apache.hadoop.fs.BlockLocation[]	locatedBlocks2Locations(java.util.List<LocatedBlock> blocks)	Convert a List to BlockLocation[]
static org.apache.hadoop.fs.BlockLocation[]	locatedBlocks2Locations(LocatedBlocks blocks)	Convert a LocatedBlocks to BlockLocations[]
static org.apache.hadoop.fs.Path	makePathFromFileId(long fileId)	Create the internal unique file path from HDFS file ID (inode ID).
static Peer	peerFromSocket(java.net.Socket socket)
static Peer	peerFromSocketAndKey(SaslDataTransferClient saslClient, java.net.Socket s, DataEncryptionKeyFactory keyFactory, org.apache.hadoop.security.token.Token<BlockTokenIdentifier> blockToken, DatanodeID datanodeId, int socketTimeoutMs)
static java.lang.String	percent2String(double percentage)	Convert percentage to a string.
static byte[]	string2Bytes(java.lang.String str)	Converts a string to a byte array using UTF8 encoding.
static java.io.InterruptedIOException	toInterruptedIOException(java.lang.String message, java.lang.InterruptedException e)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

EMPTY_BYTES

public static final byte[] EMPTY_BYTES

Constructor Detail

DFSUtilClient

public DFSUtilClient()

Method Detail

string2Bytes

public static byte[] string2Bytes(java.lang.String str)

Converts a string to a byte array using UTF8 encoding.

bytes2String

public static java.lang.String bytes2String(byte[] bytes)

Converts a byte array to a string using UTF8 encoding.

bytes2byteArray

public static byte[][] bytes2byteArray(byte[] bytes)

Converts a byte array to array of arrays of bytes on byte separator.

bytes2byteArray

public static byte[][] bytes2byteArray(byte[] bytes,
                                       int len,
                                       byte separator)

Splits first len bytes in bytes to array of arrays of bytes on byte separator.

Parameters:

bytes - the byte array to split

len - the number of bytes to split

separator - the delimiting byte

getPercentUsed

public static float getPercentUsed(long used,
                                   long capacity)

Return used as percentage of capacity

getPercentRemaining

public static float getPercentRemaining(long remaining,
                                        long capacity)

Return remaining as percentage of capacity

percent2String

public static java.lang.String percent2String(double percentage)

Convert percentage to a string.

getNameServiceIds

public static java.util.Collection<java.lang.String> getNameServiceIds(org.apache.hadoop.conf.Configuration conf)

Returns collection of nameservice Ids from the configuration.

Parameters:

conf - configuration

Returns:

collection of nameservice Ids. Empty list if unspecified.

getNameNodeIds

public static java.util.Collection<java.lang.String> getNameNodeIds(org.apache.hadoop.conf.Configuration conf,
                                                                    java.lang.String nsId)

Namenode HighAvailability related configuration. Returns collection of namenode Ids from the configuration. One logical id for each namenode in the in the HA setup.

Parameters:

conf - configuration

nsId - the nameservice ID to look at, or null for non-federated

Returns:

collection of namenode Ids

getHaNnRpcAddresses

public static java.util.Map<java.lang.String,java.util.Map<java.lang.String,java.net.InetSocketAddress>> getHaNnRpcAddresses(org.apache.hadoop.conf.Configuration conf)

Returns list of InetSocketAddress corresponding to HA NN RPC addresses from the configuration.

Parameters:

conf - configuration

Returns:

list of InetSocketAddresses

getHaNnWebHdfsAddresses

public static java.util.Map<java.lang.String,java.util.Map<java.lang.String,java.net.InetSocketAddress>> getHaNnWebHdfsAddresses(org.apache.hadoop.conf.Configuration conf,
                                                                                                                                             java.lang.String scheme)

Returns list of InetSocketAddress corresponding to HA NN HTTP addresses from the configuration.

Returns:

list of InetSocketAddresses

locatedBlocks2Locations

public static org.apache.hadoop.fs.BlockLocation[] locatedBlocks2Locations(LocatedBlocks blocks)

Convert a LocatedBlocks to BlockLocations[]

Parameters:

blocks - a LocatedBlocks

Returns:

an array of BlockLocations

locatedBlocks2Locations

public static org.apache.hadoop.fs.BlockLocation[] locatedBlocks2Locations(java.util.List<LocatedBlock> blocks)

Convert a List to BlockLocation[]

Parameters:

blocks - A List to be converted

Returns:

converted array of BlockLocation

compareBytes

public static int compareBytes(byte[] left,
                               byte[] right)

Compare two byte arrays by lexicographical order.

byteArray2bytes

public static byte[] byteArray2bytes(byte[][] pathComponents)

Given a list of path components returns a byte array

byteArray2String

public static java.lang.String byteArray2String(byte[][] pathComponents)

Given a list of path components returns a string.

getAddresses

public static java.util.Map<java.lang.String,java.util.Map<java.lang.String,java.net.InetSocketAddress>> getAddresses(org.apache.hadoop.conf.Configuration conf,
                                                                                                                                  java.lang.String defaultAddress,
                                                                                                                                  java.lang.String... keys)

Returns the configured address for all NameNodes in the cluster.

Parameters:

conf - configuration

defaultAddress - default address to return in case key is not found.

keys - Set of keys to look for in the order of preference

Returns:

a map(nameserviceId to map(namenodeId to InetSocketAddress))

getResolvedAddressesForNnId

public static java.util.Map<java.lang.String,java.net.InetSocketAddress> getResolvedAddressesForNnId(org.apache.hadoop.conf.Configuration conf,
                                                                                                           java.lang.String nsId,
                                                                                                           java.lang.String nnId,
                                                                                                           org.apache.hadoop.net.DomainNameResolver dnr,
                                                                                                           java.lang.String defaultValue,
                                                                                                           java.lang.String... keys)

getAddressesForNameserviceId

public static java.util.Map<java.lang.String,java.net.InetSocketAddress> getAddressesForNameserviceId(org.apache.hadoop.conf.Configuration conf,
                                                                                                            java.lang.String nsId,
                                                                                                            java.lang.String defaultValue,
                                                                                                            java.lang.String... keys)

getConfValue

public static java.lang.String getConfValue(java.lang.String defaultValue,
                                            java.lang.String keySuffix,
                                            org.apache.hadoop.conf.Configuration conf,
                                            java.lang.String... keys)

Given a list of keys in the order of preference, returns a value for the key in the given order from the configuration.

Parameters:

defaultValue - default value to return, when key was not found

keySuffix - suffix to add to the key, if it is not null

conf - Configuration

keys - list of keys in the order of preference

Returns:

value of the key or default if a key was not found in configuration

isValidName

public static boolean isValidName(java.lang.String src)

Whether the pathname is valid. Currently prohibits relative paths, names which contain a ":" or "//", or other non-canonical paths.

durationToString

public static java.lang.String durationToString(long durationMs)

Converts a time duration in milliseconds into DDD:HH:MM:SS format.

dateToIso8601String

public static java.lang.String dateToIso8601String(java.util.Date date)

Converts a Date into an ISO-8601 formatted datetime string.

isLocalAddress

public static boolean isLocalAddress(java.net.InetSocketAddress targetAddr)
                              throws java.io.IOException

Throws:

java.io.IOException

createClientDatanodeProtocolProxy

public static ClientDatanodeProtocol createClientDatanodeProtocolProxy(DatanodeID datanodeid,
                                                                       org.apache.hadoop.conf.Configuration conf,
                                                                       int socketTimeout,
                                                                       boolean connectToDnViaHostname,
                                                                       LocatedBlock locatedBlock)
                                                                throws java.io.IOException

Create a ClientDatanodeProtocol proxy

Throws:

java.io.IOException

createClientDatanodeProtocolProxy

public static ClientDatanodeProtocol createClientDatanodeProtocolProxy(DatanodeID datanodeid,
                                                                       org.apache.hadoop.conf.Configuration conf,
                                                                       int socketTimeout,
                                                                       boolean connectToDnViaHostname)
                                                                throws java.io.IOException

Create ClientDatanodeProtocol proxy using kerberos ticket

Throws:

java.io.IOException

createClientDatanodeProtocolProxy

public static ClientDatanodeProtocol createClientDatanodeProtocolProxy(java.net.InetSocketAddress addr,
                                                                       org.apache.hadoop.security.UserGroupInformation ticket,
                                                                       org.apache.hadoop.conf.Configuration conf,
                                                                       javax.net.SocketFactory factory)
                                                                throws java.io.IOException

Create a ClientDatanodeProtocol proxy

Throws:

java.io.IOException

createReconfigurationProtocolProxy

public static ReconfigurationProtocol createReconfigurationProtocolProxy(java.net.InetSocketAddress addr,
                                                                         org.apache.hadoop.security.UserGroupInformation ticket,
                                                                         org.apache.hadoop.conf.Configuration conf,
                                                                         javax.net.SocketFactory factory)
                                                                  throws java.io.IOException

Throws:

java.io.IOException

peerFromSocket

public static Peer peerFromSocket(java.net.Socket socket)
                           throws java.io.IOException

Throws:

java.io.IOException

peerFromSocketAndKey

public static Peer peerFromSocketAndKey(SaslDataTransferClient saslClient,
                                        java.net.Socket s,
                                        DataEncryptionKeyFactory keyFactory,
                                        org.apache.hadoop.security.token.Token<BlockTokenIdentifier> blockToken,
                                        DatanodeID datanodeId,
                                        int socketTimeoutMs)
                                 throws java.io.IOException

Throws:

java.io.IOException

getIoFileBufferSize

public static int getIoFileBufferSize(org.apache.hadoop.conf.Configuration conf)

getSmallBufferSize

public static int getSmallBufferSize(org.apache.hadoop.conf.Configuration conf)

isHDFSEncryptionEnabled

public static boolean isHDFSEncryptionEnabled(org.apache.hadoop.conf.Configuration conf)

Probe for HDFS Encryption being enabled; this uses the value of the option CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH , returning true if that property contains a non-empty, non-whitespace string.

Parameters:

conf - configuration to probe

Returns:

true if encryption is considered enabled.

getNNAddress

public static java.net.InetSocketAddress getNNAddress(java.lang.String address)

getNNAddress

public static java.net.InetSocketAddress getNNAddress(org.apache.hadoop.conf.Configuration conf)

getNNAddress

public static java.net.InetSocketAddress getNNAddress(java.net.URI filesystemURI)

Returns:

address of file system

getNNAddressCheckLogical

public static java.net.InetSocketAddress getNNAddressCheckLogical(org.apache.hadoop.conf.Configuration conf,
                                                                  java.net.URI filesystemURI)

Get the NN address from the URI. If the uri is logical, default address is returned. Otherwise return the DNS-resolved address of the URI.

Parameters:

conf - configuration

filesystemURI - URI of the file system

Returns:

address of file system

getNNUri

public static java.net.URI getNNUri(java.net.InetSocketAddress namenode)

toInterruptedIOException

public static java.io.InterruptedIOException toInterruptedIOException(java.lang.String message,
                                                                      java.lang.InterruptedException e)

connectToDN

public static IOStreamPair connectToDN(DatanodeInfo dn,
                                       int timeout,
                                       org.apache.hadoop.conf.Configuration conf,
                                       SaslDataTransferClient saslClient,
                                       javax.net.SocketFactory socketFactory,
                                       boolean connectToDnViaHostname,
                                       DataEncryptionKeyFactory dekFactory,
                                       org.apache.hadoop.security.token.Token<BlockTokenIdentifier> blockToken)
                                throws java.io.IOException

Connect to the given datanode's datantrasfer port, and return the resulting IOStreamPair. This includes encryption wrapping, etc.

Throws:

java.io.IOException

getThreadPoolExecutor

public static java.util.concurrent.ThreadPoolExecutor getThreadPoolExecutor(int corePoolSize,
                                                                            int maxPoolSize,
                                                                            long keepAliveTimeSecs,
                                                                            java.lang.String threadNamePrefix,
                                                                            boolean runRejectedExec)

Utility to create a ThreadPoolExecutor.

Parameters:

corePoolSize - - min threads in the pool, even if idle

maxPoolSize - - max threads in the pool

keepAliveTimeSecs - - max seconds beyond which excess idle threads will be terminated

threadNamePrefix - - name prefix for the pool threads

runRejectedExec - - when true, rejected tasks from ThreadPoolExecutor are run in the context of calling thread

Returns:

ThreadPoolExecutor

getThreadPoolExecutor

public static java.util.concurrent.ThreadPoolExecutor getThreadPoolExecutor(int corePoolSize,
                                                                            int maxPoolSize,
                                                                            long keepAliveTimeSecs,
                                                                            java.util.concurrent.BlockingQueue<java.lang.Runnable> queue,
                                                                            java.lang.String threadNamePrefix,
                                                                            boolean runRejectedExec)

Utility to create a ThreadPoolExecutor.

Parameters:

corePoolSize - - min threads in the pool, even if idle

maxPoolSize - - max threads in the pool

keepAliveTimeSecs - - max seconds beyond which excess idle threads will be terminated

queue - - the queue to use for holding tasks before they are executed.

threadNamePrefix - - name prefix for the pool threads

runRejectedExec - - when true, rejected tasks from ThreadPoolExecutor are run in the context of calling thread

Returns:

ThreadPoolExecutor

makePathFromFileId

public static org.apache.hadoop.fs.Path makePathFromFileId(long fileId)

Create the internal unique file path from HDFS file ID (inode ID). Unlike a regular file path, this one is guaranteed to refer to the same file at all times, across overwrites, etc.

Parameters:

fileId - File ID.

Returns:

The internal ID-based path.

getHomeDirectory

public static java.lang.String getHomeDirectory(org.apache.hadoop.conf.Configuration conf,
                                                org.apache.hadoop.security.UserGroupInformation ugi)

Returns current user home directory under a home directory prefix. The home directory prefix can be defined by HdfsClientConfigKeys.DFS_USER_HOME_DIR_PREFIX_KEY. User info is obtained from given UserGroupInformation.

Parameters:

conf - configuration

ugi - UserGroupInformation of current user.

Returns:

the home directory of current user.

getTrashRoot

public static java.lang.String getTrashRoot(org.apache.hadoop.conf.Configuration conf,
                                            org.apache.hadoop.security.UserGroupInformation ugi)

Returns trash root in non-encryption zone.

Parameters:

conf - configuration.

ugi - user of trash owner.

Returns:

unqualified path of trash root.

getEZTrashRoot

public static java.lang.String getEZTrashRoot(EncryptionZone ez,
                                              org.apache.hadoop.security.UserGroupInformation ugi)

Returns trash root in encryption zone.

Parameters:

ez - encryption zone.

ugi - user of trash owner.

Returns:

unqualified path of trash root.

getSnapshotTrashRoot

public static java.lang.String getSnapshotTrashRoot(java.lang.String ssRoot,
                                                    org.apache.hadoop.security.UserGroupInformation ugi)

Returns trash root in a snapshottable directory.

Parameters:

ssRoot - String of path to a snapshottable directory root.

ugi - user of trash owner.

Returns:

unqualified path of trash root.

isValidSnapshotName

public static boolean isValidSnapshotName(java.lang.String snapshotName)

Returns true if the name of snapshot is vlaid.

Parameters:

snapshotName - name of the snapshot.

Returns:

true if the name of snapshot is vlaid.

getSnapshotDiffReport

public static SnapshotDiffReport getSnapshotDiffReport(java.lang.String snapshotDir,
                                                       java.lang.String fromSnapshot,
                                                       java.lang.String toSnapshot,
                                                       DFSUtilClient.SnapshotDiffReportFunction withoutListing,
                                                       DFSUtilClient.SnapshotDiffReportListingFunction withListing)
                                                throws java.io.IOException

Throws:

java.io.IOException