Package org.apache.hadoop.mapreduce.v2.hs

Class HistoryFileManager

java.lang.Object

org.apache.hadoop.service.AbstractService

org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager

All Implemented Interfaces:

java.io.Closeable, java.lang.AutoCloseable, org.apache.hadoop.service.Service

@Public
@Unstable
public class HistoryFileManager
extends org.apache.hadoop.service.AbstractService

This class provides a way to interact with history files in a thread safe manor.

Nested Class Summary

Nested Classes

Modifier and Type	Class	Description
class	HistoryFileManager.HistoryFileInfo

Nested classes/interfaces inherited from interface org.apache.hadoop.service.Service

org.apache.hadoop.service.Service.STATE

Field Summary

Fields

Modifier and Type	Field	Description
protected org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager.JobListCache	jobListCache
protected java.util.concurrent.ThreadPoolExecutor	moveToDoneExecutor

Constructor Summary

Constructors

Constructor	Description
HistoryFileManager()

Method Summary

Modifier and Type	Method	Description
protected HistoryFileManager.HistoryFileInfo	createHistoryFileInfo(org.apache.hadoop.fs.Path historyFile, org.apache.hadoop.fs.Path confFile, org.apache.hadoop.fs.Path summaryFile, org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo jobIndexInfo, boolean isInDone)
protected org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager.JobListCache	createJobListCache()
protected java.util.concurrent.ThreadPoolExecutor	createMoveToDoneThreadPool(int numMoveThreads)
protected boolean	deleteDir(org.apache.hadoop.fs.FileStatus serialDir)
protected java.util.List<org.apache.hadoop.fs.FileStatus>	findTimestampedDirectories()	Finds all history directories with a timestamp component by scanning the filesystem.
java.util.Collection<HistoryFileManager.HistoryFileInfo>	getAllFileInfo()
HistoryFileManager.HistoryFileInfo	getFileInfo(org.apache.hadoop.mapreduce.v2.api.records.JobId jobId)
protected static java.util.List<org.apache.hadoop.fs.FileStatus>	scanDirectory(org.apache.hadoop.fs.Path path, org.apache.hadoop.fs.FileContext fc, org.apache.hadoop.fs.PathFilter pathFilter)
protected java.util.List<org.apache.hadoop.fs.FileStatus>	scanDirectoryForHistoryFiles(org.apache.hadoop.fs.Path path, org.apache.hadoop.fs.FileContext fc)
protected void	serviceInit(org.apache.hadoop.conf.Configuration conf)
void	serviceStop()

Methods inherited from class org.apache.hadoop.service.AbstractService

close, getBlockers, getConfig, getFailureCause, getFailureState, getLifecycleHistory, getName, getServiceState, getStartTime, init, isInState, noteFailure, putBlocker, registerGlobalListener, registerServiceListener, removeBlocker, serviceStart, setConfig, start, stop, toString, unregisterGlobalListener, unregisterServiceListener, waitForServiceToStop

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, wait, wait, wait

Field Detail

jobListCache

protected org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager.JobListCache jobListCache

moveToDoneExecutor

@VisibleForTesting
protected java.util.concurrent.ThreadPoolExecutor moveToDoneExecutor

Constructor Detail

HistoryFileManager

public HistoryFileManager()

Method Detail

serviceInit

protected void serviceInit(org.apache.hadoop.conf.Configuration conf)
                    throws java.lang.Exception

Overrides:

serviceInit in class org.apache.hadoop.service.AbstractService

Throws:

java.lang.Exception

createMoveToDoneThreadPool

protected java.util.concurrent.ThreadPoolExecutor createMoveToDoneThreadPool(int numMoveThreads)

serviceStop

public void serviceStop()
                 throws java.lang.Exception

Overrides:

serviceStop in class org.apache.hadoop.service.AbstractService

Throws:

java.lang.Exception

createJobListCache

protected org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager.JobListCache createJobListCache()

createHistoryFileInfo

protected HistoryFileManager.HistoryFileInfo createHistoryFileInfo(org.apache.hadoop.fs.Path historyFile,
                                                                   org.apache.hadoop.fs.Path confFile,
                                                                   org.apache.hadoop.fs.Path summaryFile,
                                                                   org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo jobIndexInfo,
                                                                   boolean isInDone)

scanDirectory

@VisibleForTesting
protected static java.util.List<org.apache.hadoop.fs.FileStatus> scanDirectory(org.apache.hadoop.fs.Path path,
                                                                               org.apache.hadoop.fs.FileContext fc,
                                                                               org.apache.hadoop.fs.PathFilter pathFilter)
                                                                        throws java.io.IOException

Throws:

java.io.IOException

scanDirectoryForHistoryFiles

protected java.util.List<org.apache.hadoop.fs.FileStatus> scanDirectoryForHistoryFiles(org.apache.hadoop.fs.Path path,
                                                                                       org.apache.hadoop.fs.FileContext fc)
                                                                                throws java.io.IOException

Throws:

java.io.IOException

findTimestampedDirectories

protected java.util.List<org.apache.hadoop.fs.FileStatus> findTimestampedDirectories()
                                                                              throws java.io.IOException

Finds all history directories with a timestamp component by scanning the filesystem. Used when the JobHistory server is started.

Returns:

list of history directories

Throws:

java.io.IOException

getAllFileInfo

public java.util.Collection<HistoryFileManager.HistoryFileInfo> getAllFileInfo()
                                                                        throws java.io.IOException

Throws:

java.io.IOException

getFileInfo

public HistoryFileManager.HistoryFileInfo getFileInfo(org.apache.hadoop.mapreduce.v2.api.records.JobId jobId)
                                               throws java.io.IOException

Throws:

java.io.IOException

deleteDir

protected boolean deleteDir(org.apache.hadoop.fs.FileStatus serialDir)
                     throws org.apache.hadoop.security.AccessControlException,
                            java.io.FileNotFoundException,
                            org.apache.hadoop.fs.UnsupportedFileSystemException,
                            java.io.IOException

Throws:

org.apache.hadoop.security.AccessControlException

java.io.FileNotFoundException

org.apache.hadoop.fs.UnsupportedFileSystemException

java.io.IOException