Package org.apache.hadoop.yarn.logaggregation.filecontroller

Class LogAggregationFileController

java.lang.Object

org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileController

Direct Known Subclasses:

LogAggregationIndexedFileController, LogAggregationNodeLocalTFileController, LogAggregationTFileController

@Public
@Unstable
public abstract class LogAggregationFileController
extends java.lang.Object

Base class to implement Log Aggregation File Controller.

Field Summary

Fields

Modifier and Type	Field	Description
protected static org.apache.hadoop.fs.permission.FsPermission	APP_DIR_PERMISSIONS	Permissions for the Application directory.
protected static org.apache.hadoop.fs.permission.FsPermission	APP_LOG_FILE_UMASK	Umask for the log file.
protected org.apache.hadoop.conf.Configuration	conf
protected java.lang.String	fileControllerName
protected boolean	fsSupportsChmod
protected int	maxRetry
protected java.lang.String	remoteOlderRootLogDirSuffix
protected org.apache.hadoop.fs.Path	remoteRootLogDir
protected java.lang.String	remoteRootLogDirSuffix
protected int	retentionSize
protected long	retryTimeout
protected static org.apache.hadoop.fs.permission.FsPermission	TLDIR_PERMISSIONS	Permissions for the top level directory under which app directories will be created.
protected org.apache.hadoop.security.authorize.UsersACLsManager	usersAclsManager

Constructor Summary

Constructors

Constructor	Description
LogAggregationFileController()

Method Summary

Modifier and Type	Method	Description
protected java.lang.String	aggregatedLogSuffix(java.lang.String fileName)	Create the aggregated log suffix.
protected boolean	belongsToAppAttempt(org.apache.hadoop.yarn.api.records.ApplicationAttemptId appAttemptId, java.lang.String containerIdStr)
protected boolean	checkExists(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path path, org.apache.hadoop.fs.permission.FsPermission fsPerm)
protected void	cleanOldLogs(org.apache.hadoop.fs.Path remoteNodeLogFileForApp, org.apache.hadoop.yarn.api.records.NodeId nodeId, org.apache.hadoop.security.UserGroupInformation userUgi)
protected void	closePrintStream(java.io.OutputStream out)
abstract void	closeWriter()	Close the writer.
void	createAppDir(java.lang.String user, org.apache.hadoop.yarn.api.records.ApplicationId appId, org.apache.hadoop.security.UserGroupInformation userUgi)	Create remote Application directory for log aggregation.
protected void	createDir(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path path, org.apache.hadoop.fs.permission.FsPermission fsPerm)
protected void	extractRemoteOlderRootLogDirSuffix()	Sets the remoteRootOlderLogDirSuffix class variable extracting YarnConfiguration.LOG_AGGREGATION_REMOTE_APP_LOG_DIR_SUFFIX_FMT from the configuration, or YarnConfiguration.NM_REMOTE_APP_LOG_DIR_SUFFIX
protected void	extractRemoteRootLogDir()	Sets the remoteRootLogDir class variable extracting YarnConfiguration.LOG_AGGREGATION_REMOTE_APP_LOG_DIR_FMT from the configuration or YarnConfiguration.NM_REMOTE_APP_LOG_DIR, if the former is not set.
protected void	extractRemoteRootLogDirSuffix()	Sets the remoteRootLogDirSuffix class variable extracting YarnConfiguration.LOG_AGGREGATION_REMOTE_APP_LOG_DIR_SUFFIX_FMT from the configuration, or YarnConfiguration.NM_REMOTE_APP_LOG_DIR_SUFFIX appended by the FileController's name, if the former is not set.
abstract java.util.Map<org.apache.hadoop.yarn.api.records.ApplicationAccessType,java.lang.String>	getApplicationAcls(org.apache.hadoop.fs.Path aggregatedLogPath, org.apache.hadoop.yarn.api.records.ApplicationId appId)	Returns ACLs for the application.
org.apache.hadoop.fs.RemoteIterator<org.apache.hadoop.fs.FileStatus>	getApplicationDirectoriesOfUser(java.lang.String user)	Gets all application directories of a user.
abstract java.lang.String	getApplicationOwner(org.apache.hadoop.fs.Path aggregatedLogPath, org.apache.hadoop.yarn.api.records.ApplicationId appId)	Returns the owner of the application.
java.lang.String	getFileControllerName()	Get the name of the file controller.
protected org.apache.hadoop.fs.FileSystem	getFileSystem(org.apache.hadoop.conf.Configuration conf)
java.util.Map<java.lang.String,java.util.List<ContainerLogFileInfo>>	getLogMetaFilesOfNode(ExtendedLogMetaRequest logRequest, org.apache.hadoop.fs.FileStatus currentNodeFile, org.apache.hadoop.yarn.api.records.ApplicationId appId)	Returns log file metadata for a node grouped by containers.
org.apache.hadoop.fs.RemoteIterator<org.apache.hadoop.fs.FileStatus>	getNodeFilesOfApplicationDirectory(org.apache.hadoop.fs.FileStatus appDir)	Gets all node files in an application directory.
org.apache.hadoop.fs.Path	getOlderRemoteAppLogDir(org.apache.hadoop.yarn.api.records.ApplicationId appId, java.lang.String appOwner)	Get the older remote application directory for log aggregation.
org.apache.hadoop.fs.Path	getRemoteAppLogDir(org.apache.hadoop.yarn.api.records.ApplicationId appId, java.lang.String appOwner)	Get the remote application directory for log aggregation.
org.apache.hadoop.fs.Path	getRemoteNodeLogFileForApp(org.apache.hadoop.yarn.api.records.ApplicationId appId, java.lang.String user, org.apache.hadoop.yarn.api.records.NodeId nodeId)	Get the remote aggregated log path.
java.lang.String	getRemoteOlderRootLogDirSuffix()	Get the log aggregation directory suffix.
org.apache.hadoop.fs.Path	getRemoteRootLogDir()	Get the remote root log directory.
java.lang.String	getRemoteRootLogDirSuffix()	Get the log aggregation directory suffix.
void	initialize(org.apache.hadoop.conf.Configuration conf, java.lang.String controllerName)	Initialize the log file controller.
abstract void	initializeWriter(LogAggregationFileControllerContext context)	Initialize the writer.
protected abstract void	initInternal(org.apache.hadoop.conf.Configuration conf)	Derived classes initialize themselves using this method.
boolean	isFsSupportsChmod()
abstract void	postWrite(LogAggregationFileControllerContext record)	Operations needed after write the log content.
abstract boolean	readAggregatedLogs(ContainerLogsRequest logRequest, java.io.OutputStream os)	Output container log.
abstract java.util.List<ContainerLogMeta>	readAggregatedLogsMeta(ContainerLogsRequest logRequest)	Return a list of ContainerLogMeta for an application from Remote FileSystem.
abstract void	renderAggregatedLogsBlock(HtmlBlock.Block html, View.ViewContext context)	Render Aggregated Logs block.
void	verifyAndCreateRemoteLogDir()	Verify and create the remote log directory.
abstract void	write(AggregatedLogFormat.LogKey logKey, AggregatedLogFormat.LogValue logValue)	Write the log content.

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

TLDIR_PERMISSIONS

protected static final org.apache.hadoop.fs.permission.FsPermission TLDIR_PERMISSIONS

Permissions for the top level directory under which app directories will be created.

APP_DIR_PERMISSIONS

protected static final org.apache.hadoop.fs.permission.FsPermission APP_DIR_PERMISSIONS

Permissions for the Application directory.

APP_LOG_FILE_UMASK

protected static final org.apache.hadoop.fs.permission.FsPermission APP_LOG_FILE_UMASK

Umask for the log file.

conf

protected org.apache.hadoop.conf.Configuration conf

remoteRootLogDir

protected org.apache.hadoop.fs.Path remoteRootLogDir

remoteRootLogDirSuffix

protected java.lang.String remoteRootLogDirSuffix

remoteOlderRootLogDirSuffix

protected java.lang.String remoteOlderRootLogDirSuffix

retentionSize

protected int retentionSize

fileControllerName

protected java.lang.String fileControllerName

fsSupportsChmod

protected boolean fsSupportsChmod

usersAclsManager

protected org.apache.hadoop.security.authorize.UsersACLsManager usersAclsManager

maxRetry

protected int maxRetry

retryTimeout

protected long retryTimeout

Constructor Detail

LogAggregationFileController

public LogAggregationFileController()

Method Detail

initialize

public void initialize(org.apache.hadoop.conf.Configuration conf,
                       java.lang.String controllerName)

Initialize the log file controller.

Parameters:

conf - the Configuration

controllerName - the log controller class name

initInternal

protected abstract void initInternal(org.apache.hadoop.conf.Configuration conf)

Derived classes initialize themselves using this method.

Parameters:

conf - the Configuration

getRemoteRootLogDir

public org.apache.hadoop.fs.Path getRemoteRootLogDir()

Get the remote root log directory.

Returns:

the remote root log directory path

getRemoteRootLogDirSuffix

public java.lang.String getRemoteRootLogDirSuffix()

Get the log aggregation directory suffix.

Returns:

the log aggregation directory suffix

getFileControllerName

public java.lang.String getFileControllerName()

Get the name of the file controller.

Returns:

name of the file controller.

getRemoteOlderRootLogDirSuffix

public java.lang.String getRemoteOlderRootLogDirSuffix()

Get the log aggregation directory suffix.

Returns:

the log aggregation directory suffix

initializeWriter

public abstract void initializeWriter(LogAggregationFileControllerContext context)
                               throws java.io.IOException

Initialize the writer.

Parameters:

context - the LogAggregationFileControllerContext

Throws:

java.io.IOException - if fails to initialize the writer

closeWriter

public abstract void closeWriter()
                          throws LogAggregationDFSException

Close the writer.

Throws:

LogAggregationDFSException - if the closing of the writer fails (for example due to HDFS quota being exceeded)

write

public abstract void write(AggregatedLogFormat.LogKey logKey,
                           AggregatedLogFormat.LogValue logValue)
                    throws java.io.IOException

Write the log content.

Parameters:

logKey - the log key

logValue - the log content

Throws:

java.io.IOException - if fails to write the logs

postWrite

public abstract void postWrite(LogAggregationFileControllerContext record)
                        throws java.lang.Exception

Operations needed after write the log content.

Parameters:

record - the LogAggregationFileControllerContext

Throws:

java.lang.Exception - if anything fails

closePrintStream

protected void closePrintStream(java.io.OutputStream out)

readAggregatedLogs

public abstract boolean readAggregatedLogs(ContainerLogsRequest logRequest,
                                           java.io.OutputStream os)
                                    throws java.io.IOException

Output container log.

Parameters:

logRequest - ContainerLogsRequest

os - the output stream

Returns:

true if we can read the aggregated logs successfully

Throws:

java.io.IOException - if we can not access the log file.

readAggregatedLogsMeta

public abstract java.util.List<ContainerLogMeta> readAggregatedLogsMeta(ContainerLogsRequest logRequest)
                                                                 throws java.io.IOException

Return a list of ContainerLogMeta for an application from Remote FileSystem.

Parameters:

logRequest - ContainerLogsRequest

Returns:

a list of ContainerLogMeta

Throws:

java.io.IOException - if there is no available log file

getLogMetaFilesOfNode

public java.util.Map<java.lang.String,java.util.List<ContainerLogFileInfo>> getLogMetaFilesOfNode(ExtendedLogMetaRequest logRequest,
                                                                                                        org.apache.hadoop.fs.FileStatus currentNodeFile,
                                                                                                        org.apache.hadoop.yarn.api.records.ApplicationId appId)
                                                                                                 throws java.io.IOException

Returns log file metadata for a node grouped by containers.

Parameters:

logRequest - extended query information holder

currentNodeFile - file status of a node in an application directory

appId - id of the application, which is the same as in node path

Returns:

log file metadata

Throws:

java.io.IOException - if there is no node file

getApplicationDirectoriesOfUser

public org.apache.hadoop.fs.RemoteIterator<org.apache.hadoop.fs.FileStatus> getApplicationDirectoriesOfUser(java.lang.String user)
                                                                                                     throws java.io.IOException

Gets all application directories of a user.

Parameters:

user - name of the user

Returns:

a lazy iterator of directories

Throws:

java.io.IOException - if user directory does not exist

getNodeFilesOfApplicationDirectory

public org.apache.hadoop.fs.RemoteIterator<org.apache.hadoop.fs.FileStatus> getNodeFilesOfApplicationDirectory(org.apache.hadoop.fs.FileStatus appDir)
                                                                                                        throws java.io.IOException

Gets all node files in an application directory.

Parameters:

appDir - application directory

Returns:

a lazy iterator of files

Throws:

java.io.IOException - if file context is not reachable

renderAggregatedLogsBlock

public abstract void renderAggregatedLogsBlock(HtmlBlock.Block html,
                                               View.ViewContext context)

Render Aggregated Logs block.

Parameters:

html - the html

context - the ViewContext

getApplicationOwner

public abstract java.lang.String getApplicationOwner(org.apache.hadoop.fs.Path aggregatedLogPath,
                                                     org.apache.hadoop.yarn.api.records.ApplicationId appId)
                                              throws java.io.IOException

Returns the owner of the application.

Parameters:

aggregatedLogPath - the aggregatedLog path

appId - the ApplicationId

Returns:

the application owner

Throws:

java.io.IOException - if we can not get the application owner

getApplicationAcls

public abstract java.util.Map<org.apache.hadoop.yarn.api.records.ApplicationAccessType,java.lang.String> getApplicationAcls(org.apache.hadoop.fs.Path aggregatedLogPath,
                                                                                                                                  org.apache.hadoop.yarn.api.records.ApplicationId appId)
                                                                                                                           throws java.io.IOException

Returns ACLs for the application. An empty map is returned if no ACLs are found.

Parameters:

aggregatedLogPath - the aggregatedLog path.

appId - the ApplicationId

Returns:

a map of the Application ACLs.

Throws:

java.io.IOException - if we can not get the application acls

extractRemoteRootLogDirSuffix

protected void extractRemoteRootLogDirSuffix()

Sets the remoteRootLogDirSuffix class variable extracting YarnConfiguration.LOG_AGGREGATION_REMOTE_APP_LOG_DIR_SUFFIX_FMT from the configuration, or YarnConfiguration.NM_REMOTE_APP_LOG_DIR_SUFFIX appended by the FileController's name, if the former is not set.

extractRemoteOlderRootLogDirSuffix

protected void extractRemoteOlderRootLogDirSuffix()

Sets the remoteRootOlderLogDirSuffix class variable extracting YarnConfiguration.LOG_AGGREGATION_REMOTE_APP_LOG_DIR_SUFFIX_FMT from the configuration, or YarnConfiguration.NM_REMOTE_APP_LOG_DIR_SUFFIX

extractRemoteRootLogDir

protected void extractRemoteRootLogDir()

Sets the remoteRootLogDir class variable extracting YarnConfiguration.LOG_AGGREGATION_REMOTE_APP_LOG_DIR_FMT from the configuration or YarnConfiguration.NM_REMOTE_APP_LOG_DIR, if the former is not set.

verifyAndCreateRemoteLogDir

public void verifyAndCreateRemoteLogDir()

Verify and create the remote log directory.

createAppDir

public void createAppDir(java.lang.String user,
                         org.apache.hadoop.yarn.api.records.ApplicationId appId,
                         org.apache.hadoop.security.UserGroupInformation userUgi)

Create remote Application directory for log aggregation.

Parameters:

user - the user

appId - the application ID

userUgi - the UGI

getFileSystem

@VisibleForTesting
protected org.apache.hadoop.fs.FileSystem getFileSystem(org.apache.hadoop.conf.Configuration conf)
                                                 throws java.io.IOException

Throws:

java.io.IOException

createDir

protected void createDir(org.apache.hadoop.fs.FileSystem fs,
                         org.apache.hadoop.fs.Path path,
                         org.apache.hadoop.fs.permission.FsPermission fsPerm)
                  throws java.io.IOException

Throws:

java.io.IOException

checkExists

protected boolean checkExists(org.apache.hadoop.fs.FileSystem fs,
                              org.apache.hadoop.fs.Path path,
                              org.apache.hadoop.fs.permission.FsPermission fsPerm)
                       throws java.io.IOException

Throws:

java.io.IOException

getRemoteNodeLogFileForApp

public org.apache.hadoop.fs.Path getRemoteNodeLogFileForApp(org.apache.hadoop.yarn.api.records.ApplicationId appId,
                                                            java.lang.String user,
                                                            org.apache.hadoop.yarn.api.records.NodeId nodeId)

Get the remote aggregated log path.

Parameters:

appId - the ApplicationId

user - the Application Owner

nodeId - the NodeManager Id

Returns:

the remote aggregated log path

getRemoteAppLogDir

public org.apache.hadoop.fs.Path getRemoteAppLogDir(org.apache.hadoop.yarn.api.records.ApplicationId appId,
                                                    java.lang.String appOwner)
                                             throws java.io.IOException

Get the remote application directory for log aggregation.

Parameters:

appId - the Application ID

appOwner - the Application Owner

Returns:

the remote application directory

Throws:

java.io.IOException - if can not find the remote application directory

getOlderRemoteAppLogDir

public org.apache.hadoop.fs.Path getOlderRemoteAppLogDir(org.apache.hadoop.yarn.api.records.ApplicationId appId,
                                                         java.lang.String appOwner)
                                                  throws java.io.IOException

Get the older remote application directory for log aggregation.

Parameters:

appId - the Application ID

appOwner - the Application Owner

Returns:

the older remote application directory

Throws:

java.io.IOException - if can not find the remote application directory

cleanOldLogs

protected void cleanOldLogs(org.apache.hadoop.fs.Path remoteNodeLogFileForApp,
                            org.apache.hadoop.yarn.api.records.NodeId nodeId,
                            org.apache.hadoop.security.UserGroupInformation userUgi)

aggregatedLogSuffix

protected java.lang.String aggregatedLogSuffix(java.lang.String fileName)

Create the aggregated log suffix. The LogAggregationFileController should call this to get the suffix and append the suffix to the end of each log. This would keep the aggregated log format consistent.

Parameters:

fileName - the File Name

Returns:

the aggregated log suffix String

isFsSupportsChmod

public boolean isFsSupportsChmod()

belongsToAppAttempt

protected boolean belongsToAppAttempt(org.apache.hadoop.yarn.api.records.ApplicationAttemptId appAttemptId,
                                      java.lang.String containerIdStr)