Package org.apache.hadoop.yarn.logaggregation

Class LogAggregationUtils

java.lang.Object

org.apache.hadoop.yarn.logaggregation.LogAggregationUtils

@Private
public class LogAggregationUtils
extends java.lang.Object

Field Summary

Fields

Modifier and Type	Field	Description
static org.slf4j.Logger	LOG
static java.lang.String	LOG_PATH_FOR_LOCAL_VOLUME
static java.lang.String	TMP_FILE_SUFFIX

Constructor Summary

Constructors

Constructor	Description
LogAggregationUtils()

Method Summary

Modifier and Type	Method	Description
static void	checkACLConf(boolean isUsersACLEnable, boolean forceInit, org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.Path rootDir)
static void	cleanAllACE(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.Path rootDir)	Remove all ACE from users directories at root log directory
static java.lang.String	getBucketSuffix()	Returns the bucket suffix component of the log dir.
static org.apache.hadoop.fs.RemoteIterator<org.apache.hadoop.fs.FileStatus>	getNodeFiles(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.Path remoteAppLogDir, org.apache.hadoop.yarn.api.records.ApplicationId appId, java.lang.String appOwner)	Get all available log files under remote app log directory.
static java.lang.String	getNodeString(java.lang.String nodeId)
static java.lang.String	getNodeString(org.apache.hadoop.yarn.api.records.NodeId nodeId)	Converts a nodeId to a form used in the app log file name.
static org.apache.hadoop.fs.Path	getOlderRemoteAppLogDir(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.yarn.api.records.ApplicationId appId, java.lang.String user, org.apache.hadoop.fs.Path remoteRootLogDir, java.lang.String suffix)
static org.apache.hadoop.fs.Path	getOlderRemoteAppLogDir(org.apache.hadoop.yarn.api.records.ApplicationId appId, java.lang.String user, org.apache.hadoop.fs.Path remoteRootLogDir, java.lang.String suffix)	Gets the older remote app log dir.
static org.apache.hadoop.fs.Path	getOlderRemoteLogSuffixedDir(org.apache.hadoop.fs.Path remoteRootLogDir, java.lang.String user, java.lang.String suffix)	Gets the older remote suffixed log dir for the user.
static org.apache.hadoop.fs.Path	getRemoteAppLogDir(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.yarn.api.records.ApplicationId appId, java.lang.String appOwner, org.apache.hadoop.fs.Path remoteRootLogDir, java.lang.String suffix)	Return the remote application log directory.
static org.apache.hadoop.fs.Path	getRemoteAppLogDir(org.apache.hadoop.fs.Path remoteRootLogDir, org.apache.hadoop.yarn.api.records.ApplicationId appId, java.lang.String user, java.lang.String suffix)	Gets the remote app log dir.
static org.apache.hadoop.fs.Path	getRemoteBucketDir(org.apache.hadoop.fs.Path remoteRootLogDir, java.lang.String user, java.lang.String suffix, org.apache.hadoop.yarn.api.records.ApplicationId appId)	Gets the remote log user's bucket dir.
static org.apache.hadoop.fs.RemoteIterator<org.apache.hadoop.fs.FileStatus>	getRemoteFiles(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.Path appPath)
static org.apache.hadoop.fs.Path	getRemoteLogSuffixedDir(org.apache.hadoop.fs.Path remoteRootLogDir, java.lang.String user, java.lang.String suffix)	Gets the remote suffixed log dir for the user.
static org.apache.hadoop.fs.Path	getRemoteLogUserDir(org.apache.hadoop.fs.Path remoteRootLogDir, java.lang.String user)	Gets the remote log user dir.
static org.apache.hadoop.fs.RemoteIterator<org.apache.hadoop.fs.FileStatus>	getRemoteNodeFileDir(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.yarn.api.records.ApplicationId appId, java.lang.String appOwner, org.apache.hadoop.fs.Path remoteRootLogDir, java.lang.String suffix, java.lang.String oldSuffix)	Get all available log files under remote app log directory.
static java.util.List<org.apache.hadoop.fs.FileStatus>	getRemoteNodeFileList(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.yarn.api.records.ApplicationId appId, java.lang.String appOwner, org.apache.hadoop.fs.Path remoteRootLogDir, java.lang.String suffix, java.lang.String oldSuffix)	Get all available log files under remote app log directory.
static org.apache.hadoop.fs.Path	getRemoteNodeLogFileForApp(org.apache.hadoop.fs.Path remoteRootLogDir, org.apache.hadoop.yarn.api.records.ApplicationId appId, java.lang.String user, org.apache.hadoop.yarn.api.records.NodeId nodeId, java.lang.String suffix)	Constructs the full filename for an application's log file per node.
static org.apache.hadoop.fs.RemoteIterator<org.apache.hadoop.fs.FileStatus>	getUserRemoteLogDir(org.apache.hadoop.conf.Configuration conf, java.lang.String user, org.apache.hadoop.fs.Path remoteRootLogDir, java.lang.String remoteRootLogDirSuffix)
static void	initACLForAggregatedLogs(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.security.authorize.UsersACLsManager usersAclsManager, org.apache.hadoop.fs.Path rootDir)	Set ACE to users log aggregated directories at root log directory based on user ACL mapping
static boolean	isOlderPathEnabled(org.apache.hadoop.conf.Configuration conf)	Check if older Application Log Directory has to be included.

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

LOG

public static final org.slf4j.Logger LOG

TMP_FILE_SUFFIX

public static final java.lang.String TMP_FILE_SUFFIX

See Also:

Constant Field Values

LOG_PATH_FOR_LOCAL_VOLUME

public static final java.lang.String LOG_PATH_FOR_LOCAL_VOLUME

See Also:

Constant Field Values

Constructor Detail

LogAggregationUtils

public LogAggregationUtils()

Method Detail

getRemoteNodeLogFileForApp

public static org.apache.hadoop.fs.Path getRemoteNodeLogFileForApp(org.apache.hadoop.fs.Path remoteRootLogDir,
                                                                   org.apache.hadoop.yarn.api.records.ApplicationId appId,
                                                                   java.lang.String user,
                                                                   org.apache.hadoop.yarn.api.records.NodeId nodeId,
                                                                   java.lang.String suffix)

Constructs the full filename for an application's log file per node.

Parameters:

remoteRootLogDir - the aggregated remote root log dir

appId - the application Id

user - the application owner

nodeId - the node id

suffix - the log dir suffix

Returns:

the remote log file.

getRemoteAppLogDir

public static org.apache.hadoop.fs.Path getRemoteAppLogDir(org.apache.hadoop.fs.Path remoteRootLogDir,
                                                           org.apache.hadoop.yarn.api.records.ApplicationId appId,
                                                           java.lang.String user,
                                                           java.lang.String suffix)

Gets the remote app log dir.

Parameters:

remoteRootLogDir - the aggregated log remote root log dir

appId - the application id

user - the application owner

suffix - the log directory suffix

Returns:

the remote application specific log dir.

getOlderRemoteAppLogDir

public static org.apache.hadoop.fs.Path getOlderRemoteAppLogDir(org.apache.hadoop.yarn.api.records.ApplicationId appId,
                                                                java.lang.String user,
                                                                org.apache.hadoop.fs.Path remoteRootLogDir,
                                                                java.lang.String suffix)

Gets the older remote app log dir.

Parameters:

appId - the application id

user - the application owner

remoteRootLogDir - the aggregated log remote root log dir

suffix - the log directory suffix

Returns:

the remote application specific log dir.

getOlderRemoteAppLogDir

public static org.apache.hadoop.fs.Path getOlderRemoteAppLogDir(org.apache.hadoop.conf.Configuration conf,
                                                                org.apache.hadoop.yarn.api.records.ApplicationId appId,
                                                                java.lang.String user,
                                                                org.apache.hadoop.fs.Path remoteRootLogDir,
                                                                java.lang.String suffix)
                                                         throws java.io.IOException

Throws:

java.io.IOException

getRemoteLogSuffixedDir

public static org.apache.hadoop.fs.Path getRemoteLogSuffixedDir(org.apache.hadoop.fs.Path remoteRootLogDir,
                                                                java.lang.String user,
                                                                java.lang.String suffix)

Gets the remote suffixed log dir for the user.

Parameters:

remoteRootLogDir - the aggregated log remote root log dir

user - the application owner

suffix - the log dir suffix

Returns:

the remote suffixed log dir.

getOlderRemoteLogSuffixedDir

public static org.apache.hadoop.fs.Path getOlderRemoteLogSuffixedDir(org.apache.hadoop.fs.Path remoteRootLogDir,
                                                                     java.lang.String user,
                                                                     java.lang.String suffix)

Gets the older remote suffixed log dir for the user.

Parameters:

remoteRootLogDir - the aggregated log remote root log dir

user - the application owner

suffix - the log dir suffix

Returns:

the older remote suffixed log dir.

getRemoteLogUserDir

public static org.apache.hadoop.fs.Path getRemoteLogUserDir(org.apache.hadoop.fs.Path remoteRootLogDir,
                                                            java.lang.String user)

Gets the remote log user dir.

Parameters:

remoteRootLogDir - the aggregated log remote root log dir

user - the application owner

Returns:

the remote per user log dir.

getRemoteBucketDir

public static org.apache.hadoop.fs.Path getRemoteBucketDir(org.apache.hadoop.fs.Path remoteRootLogDir,
                                                           java.lang.String user,
                                                           java.lang.String suffix,
                                                           org.apache.hadoop.yarn.api.records.ApplicationId appId)

Gets the remote log user's bucket dir.

Parameters:

remoteRootLogDir - the aggregated log remote root log dir

user - the application owner

suffix - the log dir suffix

appId - the application id

Returns:

the remote log per user per cluster timestamp per bucket dir.

isOlderPathEnabled

public static boolean isOlderPathEnabled(org.apache.hadoop.conf.Configuration conf)

Check if older Application Log Directory has to be included.

Parameters:

conf - the configuration

Returns:

Is Older App Log Dir enabled?

getBucketSuffix

public static java.lang.String getBucketSuffix()

Returns the bucket suffix component of the log dir.

Returns:

the bucket suffix which appended to user log dir

getNodeString

@VisibleForTesting
public static java.lang.String getNodeString(org.apache.hadoop.yarn.api.records.NodeId nodeId)

Converts a nodeId to a form used in the app log file name.

Parameters:

nodeId - the nodeId

Returns:

the node string to be used to construct the file name.

getNodeString

@VisibleForTesting
public static java.lang.String getNodeString(java.lang.String nodeId)

getRemoteAppLogDir

public static org.apache.hadoop.fs.Path getRemoteAppLogDir(org.apache.hadoop.conf.Configuration conf,
                                                           org.apache.hadoop.yarn.api.records.ApplicationId appId,
                                                           java.lang.String appOwner,
                                                           org.apache.hadoop.fs.Path remoteRootLogDir,
                                                           java.lang.String suffix)
                                                    throws java.io.IOException

Return the remote application log directory.

Parameters:

conf - the configuration

appId - the application

appOwner - the application owner

remoteRootLogDir - the remote root log directory

suffix - the log directory suffix

Returns:

the remote application log directory path

Throws:

java.io.IOException - if we can not find remote application log directory

getNodeFiles

public static org.apache.hadoop.fs.RemoteIterator<org.apache.hadoop.fs.FileStatus> getNodeFiles(org.apache.hadoop.conf.Configuration conf,
                                                                                                org.apache.hadoop.fs.Path remoteAppLogDir,
                                                                                                org.apache.hadoop.yarn.api.records.ApplicationId appId,
                                                                                                java.lang.String appOwner)
                                                                                         throws java.io.IOException

Get all available log files under remote app log directory.

Parameters:

conf - the configuration

remoteAppLogDir - the application log directory

appId - the applicationId

appOwner - the application owner

Returns:

the iterator of available log files

Throws:

java.io.IOException - if there is no log file directory

getRemoteNodeFileDir

public static org.apache.hadoop.fs.RemoteIterator<org.apache.hadoop.fs.FileStatus> getRemoteNodeFileDir(org.apache.hadoop.conf.Configuration conf,
                                                                                                        org.apache.hadoop.yarn.api.records.ApplicationId appId,
                                                                                                        java.lang.String appOwner,
                                                                                                        org.apache.hadoop.fs.Path remoteRootLogDir,
                                                                                                        java.lang.String suffix,
                                                                                                        java.lang.String oldSuffix)
                                                                                                 throws java.io.IOException

Get all available log files under remote app log directory.

Parameters:

conf - the configuration

appId - the applicationId

appOwner - the application owner

remoteRootLogDir - the remote root log directory

suffix - the log directory suffix

Returns:

the iterator of available log files

Throws:

java.io.IOException - if there is no log file available

getRemoteNodeFileList

public static java.util.List<org.apache.hadoop.fs.FileStatus> getRemoteNodeFileList(org.apache.hadoop.conf.Configuration conf,
                                                                                    org.apache.hadoop.yarn.api.records.ApplicationId appId,
                                                                                    java.lang.String appOwner,
                                                                                    org.apache.hadoop.fs.Path remoteRootLogDir,
                                                                                    java.lang.String suffix,
                                                                                    java.lang.String oldSuffix)
                                                                             throws java.io.IOException

Get all available log files under remote app log directory.

Parameters:

conf - the configuration

appId - the applicationId

appOwner - the application owner

remoteRootLogDir - the remote root log directory

suffix - the log directory suffix

Returns:

the list of available log files

Throws:

java.io.IOException - if there is no log file available

getRemoteFiles

public static org.apache.hadoop.fs.RemoteIterator<org.apache.hadoop.fs.FileStatus> getRemoteFiles(org.apache.hadoop.conf.Configuration conf,
                                                                                                  org.apache.hadoop.fs.Path appPath)
                                                                                           throws java.io.IOException

Throws:

java.io.IOException

getUserRemoteLogDir

public static org.apache.hadoop.fs.RemoteIterator<org.apache.hadoop.fs.FileStatus> getUserRemoteLogDir(org.apache.hadoop.conf.Configuration conf,
                                                                                                       java.lang.String user,
                                                                                                       org.apache.hadoop.fs.Path remoteRootLogDir,
                                                                                                       java.lang.String remoteRootLogDirSuffix)
                                                                                                throws java.io.IOException

Throws:

java.io.IOException

initACLForAggregatedLogs

public static void initACLForAggregatedLogs(org.apache.hadoop.conf.Configuration conf,
                                            org.apache.hadoop.security.authorize.UsersACLsManager usersAclsManager,
                                            org.apache.hadoop.fs.Path rootDir)

Set ACE to users log aggregated directories at root log directory based on user ACL mapping

Parameters:

conf - the configuration

usersAclsManager - user ACL mapping

rootDir - the remote root log directory

cleanAllACE

public static void cleanAllACE(org.apache.hadoop.conf.Configuration conf,
                               org.apache.hadoop.fs.Path rootDir)

Remove all ACE from users directories at root log directory

Parameters:

conf - the configuration

rootDir - the remote root log directory

checkACLConf

public static void checkACLConf(boolean isUsersACLEnable,
                                boolean forceInit,
                                org.apache.hadoop.conf.Configuration conf,
                                org.apache.hadoop.fs.Path rootDir)