Package org.apache.hadoop.mapred

Class ShuffleHandler

java.lang.Object

org.apache.hadoop.service.AbstractService

org.apache.hadoop.yarn.server.api.AuxiliaryService

org.apache.hadoop.mapred.ShuffleHandler

All Implemented Interfaces:

java.io.Closeable, java.lang.AutoCloseable, org.apache.hadoop.service.Service

public class ShuffleHandler
extends org.apache.hadoop.yarn.server.api.AuxiliaryService

Nested Class Summary

Nested classes/interfaces inherited from interface org.apache.hadoop.service.Service

org.apache.hadoop.service.Service.STATE

Field Summary

Fields

Modifier and Type	Field	Description
protected io.netty.channel.group.ChannelGroup	allChannels
static org.slf4j.Logger	AUDITLOG
static java.lang.String	CONCURRENCY_LEVEL
static java.lang.String	CONNECTION_CLOSE
protected static org.apache.hadoop.yarn.server.records.Version	CURRENT_VERSION_INFO
static int	DEFAULT_CONCURRENCY_LEVEL
static int	DEFAULT_EXPIRE_AFTER_ACCESS_MINUTES
static int	DEFAULT_MAX_SHUFFLE_CONNECTIONS
static int	DEFAULT_MAX_SHUFFLE_THREADS
static int	DEFAULT_MAX_WEIGHT
static int	DEFAULT_SHUFFLE_BUFFER_SIZE
static boolean	DEFAULT_SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED
static int	DEFAULT_SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT
static int	DEFAULT_SHUFFLE_LISTEN_QUEUE_SIZE
static boolean	DEFAULT_SHUFFLE_MANAGE_OS_CACHE
static int	DEFAULT_SHUFFLE_MAPOUTPUT_META_INFO_CACHE_SIZE
static int	DEFAULT_SHUFFLE_MAX_SESSION_OPEN_FILES
static int	DEFAULT_SHUFFLE_PORT
static int	DEFAULT_SHUFFLE_READAHEAD_BYTES
static boolean	DEFAULT_SHUFFLE_TRANSFERTO_ALLOWED
static int	DEFAULT_SUFFLE_SSL_FILE_BUFFER_SIZE
static java.lang.String	EXPIRE_AFTER_ACCESS_MINUTES
static long	FETCH_RETRY_DELAY
static java.util.regex.Pattern	IGNORABLE_ERROR_MESSAGE
static org.slf4j.Logger	LOG
static java.lang.String	MAPREDUCE_SHUFFLE_SERVICEID
static java.lang.String	MAX_SHUFFLE_CONNECTIONS
static java.lang.String	MAX_SHUFFLE_THREADS
static java.lang.String	MAX_WEIGHT
protected org.apache.hadoop.metrics2.MetricsSystem	ms
static java.lang.String	RETRY_AFTER_HEADER
protected org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager	secretManager
static java.lang.String	SHUFFLE_BUFFER_SIZE
static java.lang.String	SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED
static java.lang.String	SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT
static java.lang.String	SHUFFLE_LISTEN_QUEUE_SIZE
static java.lang.String	SHUFFLE_MANAGE_OS_CACHE
static java.lang.String	SHUFFLE_MAPOUTPUT_META_INFO_CACHE_SIZE
static java.lang.String	SHUFFLE_MAX_SESSION_OPEN_FILES
static java.lang.String	SHUFFLE_PORT_CONFIG_KEY
static java.lang.String	SHUFFLE_READAHEAD_BYTES
static java.lang.String	SHUFFLE_TRANSFERTO_ALLOWED
static java.lang.String	SUFFLE_SSL_FILE_BUFFER_SIZE_KEY
static io.netty.handler.codec.http.HttpResponseStatus	TOO_MANY_REQ_STATUS
protected java.util.Map<java.lang.String,java.lang.String>	userRsrc
static boolean	WINDOWS_DEFAULT_SHUFFLE_TRANSFERTO_ALLOWED

Constructor Summary

Constructors

Constructor	Description
ShuffleHandler()

Method Summary

Modifier and Type	Method	Description
protected ShuffleChannelHandlerContext	createHandlerContext()
static int	deserializeMetaData(java.nio.ByteBuffer meta)	A helper function to deserialize the metadata returned by ShuffleHandler.
static org.apache.hadoop.security.token.Token<org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier>	deserializeServiceData(java.nio.ByteBuffer secret)
protected org.apache.hadoop.yarn.server.records.Version	getCurrentVersion()
java.nio.ByteBuffer	getMetaData()
void	initializeApplication(org.apache.hadoop.yarn.server.api.ApplicationInitializationContext context)
static java.nio.ByteBuffer	serializeMetaData(int port)	Serialize the shuffle port into a ByteBuffer for use later on.
static java.nio.ByteBuffer	serializeServiceData(org.apache.hadoop.security.token.Token<org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier> jobToken)	A helper function to serialize the JobTokenIdentifier to be sent to the ShuffleHandler as ServiceData.
protected void	serviceInit(org.apache.hadoop.conf.Configuration conf)
protected void	serviceStart()
protected void	serviceStop()
void	stopApplication(org.apache.hadoop.yarn.server.api.ApplicationTerminationContext context)

Methods inherited from class org.apache.hadoop.yarn.server.api.AuxiliaryService

getAuxiliaryLocalPathHandler, getMetaData, getRecoveryPath, initializeContainer, setAuxiliaryLocalPathHandler, setRecoveryPath, stopContainer

Methods inherited from class org.apache.hadoop.service.AbstractService

close, getBlockers, getConfig, getFailureCause, getFailureState, getLifecycleHistory, getName, getServiceState, getStartTime, init, isInState, noteFailure, putBlocker, registerGlobalListener, registerServiceListener, removeBlocker, setConfig, start, stop, toString, unregisterGlobalListener, unregisterServiceListener, waitForServiceToStop

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, wait, wait, wait

Field Detail

LOG

public static final org.slf4j.Logger LOG

AUDITLOG

public static final org.slf4j.Logger AUDITLOG

SHUFFLE_MANAGE_OS_CACHE

public static final java.lang.String SHUFFLE_MANAGE_OS_CACHE

See Also:

Constant Field Values

DEFAULT_SHUFFLE_MANAGE_OS_CACHE

public static final boolean DEFAULT_SHUFFLE_MANAGE_OS_CACHE

See Also:

Constant Field Values

SHUFFLE_READAHEAD_BYTES

public static final java.lang.String SHUFFLE_READAHEAD_BYTES

See Also:

Constant Field Values

DEFAULT_SHUFFLE_READAHEAD_BYTES

public static final int DEFAULT_SHUFFLE_READAHEAD_BYTES

See Also:

Constant Field Values

MAX_WEIGHT

public static final java.lang.String MAX_WEIGHT

See Also:

Constant Field Values

DEFAULT_MAX_WEIGHT

public static final int DEFAULT_MAX_WEIGHT

See Also:

Constant Field Values

EXPIRE_AFTER_ACCESS_MINUTES

public static final java.lang.String EXPIRE_AFTER_ACCESS_MINUTES

See Also:

Constant Field Values

DEFAULT_EXPIRE_AFTER_ACCESS_MINUTES

public static final int DEFAULT_EXPIRE_AFTER_ACCESS_MINUTES

See Also:

Constant Field Values

CONCURRENCY_LEVEL

public static final java.lang.String CONCURRENCY_LEVEL

See Also:

Constant Field Values

DEFAULT_CONCURRENCY_LEVEL

public static final int DEFAULT_CONCURRENCY_LEVEL

See Also:

Constant Field Values

IGNORABLE_ERROR_MESSAGE

public static final java.util.regex.Pattern IGNORABLE_ERROR_MESSAGE

CURRENT_VERSION_INFO

protected static final org.apache.hadoop.yarn.server.records.Version CURRENT_VERSION_INFO

TOO_MANY_REQ_STATUS

public static final io.netty.handler.codec.http.HttpResponseStatus TOO_MANY_REQ_STATUS

FETCH_RETRY_DELAY

public static final long FETCH_RETRY_DELAY

See Also:

Constant Field Values

RETRY_AFTER_HEADER

public static final java.lang.String RETRY_AFTER_HEADER

See Also:

Constant Field Values

allChannels

protected final io.netty.channel.group.ChannelGroup allChannels

secretManager

protected org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager secretManager

userRsrc

protected java.util.Map<java.lang.String,java.lang.String> userRsrc

MAPREDUCE_SHUFFLE_SERVICEID

public static final java.lang.String MAPREDUCE_SHUFFLE_SERVICEID

See Also:

Constant Field Values

SHUFFLE_PORT_CONFIG_KEY

public static final java.lang.String SHUFFLE_PORT_CONFIG_KEY

See Also:

Constant Field Values

DEFAULT_SHUFFLE_PORT

public static final int DEFAULT_SHUFFLE_PORT

See Also:

Constant Field Values

SHUFFLE_LISTEN_QUEUE_SIZE

public static final java.lang.String SHUFFLE_LISTEN_QUEUE_SIZE

See Also:

Constant Field Values

DEFAULT_SHUFFLE_LISTEN_QUEUE_SIZE

public static final int DEFAULT_SHUFFLE_LISTEN_QUEUE_SIZE

See Also:

Constant Field Values

SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED

public static final java.lang.String SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED

See Also:

Constant Field Values

DEFAULT_SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED

public static final boolean DEFAULT_SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED

See Also:

Constant Field Values

SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT

public static final java.lang.String SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT

See Also:

Constant Field Values

DEFAULT_SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT

public static final int DEFAULT_SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT

See Also:

Constant Field Values

SHUFFLE_MAPOUTPUT_META_INFO_CACHE_SIZE

public static final java.lang.String SHUFFLE_MAPOUTPUT_META_INFO_CACHE_SIZE

See Also:

Constant Field Values

DEFAULT_SHUFFLE_MAPOUTPUT_META_INFO_CACHE_SIZE

public static final int DEFAULT_SHUFFLE_MAPOUTPUT_META_INFO_CACHE_SIZE

See Also:

Constant Field Values

CONNECTION_CLOSE

public static final java.lang.String CONNECTION_CLOSE

See Also:

Constant Field Values

SUFFLE_SSL_FILE_BUFFER_SIZE_KEY

public static final java.lang.String SUFFLE_SSL_FILE_BUFFER_SIZE_KEY

See Also:

Constant Field Values

DEFAULT_SUFFLE_SSL_FILE_BUFFER_SIZE

public static final int DEFAULT_SUFFLE_SSL_FILE_BUFFER_SIZE

See Also:

Constant Field Values

MAX_SHUFFLE_CONNECTIONS

public static final java.lang.String MAX_SHUFFLE_CONNECTIONS

See Also:

Constant Field Values

DEFAULT_MAX_SHUFFLE_CONNECTIONS

public static final int DEFAULT_MAX_SHUFFLE_CONNECTIONS

See Also:

Constant Field Values

MAX_SHUFFLE_THREADS

public static final java.lang.String MAX_SHUFFLE_THREADS

See Also:

Constant Field Values

DEFAULT_MAX_SHUFFLE_THREADS

public static final int DEFAULT_MAX_SHUFFLE_THREADS

See Also:

Constant Field Values

SHUFFLE_BUFFER_SIZE

public static final java.lang.String SHUFFLE_BUFFER_SIZE

See Also:

Constant Field Values

DEFAULT_SHUFFLE_BUFFER_SIZE

public static final int DEFAULT_SHUFFLE_BUFFER_SIZE

See Also:

Constant Field Values

SHUFFLE_TRANSFERTO_ALLOWED

public static final java.lang.String SHUFFLE_TRANSFERTO_ALLOWED

See Also:

Constant Field Values

DEFAULT_SHUFFLE_TRANSFERTO_ALLOWED

public static final boolean DEFAULT_SHUFFLE_TRANSFERTO_ALLOWED

See Also:

Constant Field Values

WINDOWS_DEFAULT_SHUFFLE_TRANSFERTO_ALLOWED

public static final boolean WINDOWS_DEFAULT_SHUFFLE_TRANSFERTO_ALLOWED

See Also:

Constant Field Values

SHUFFLE_MAX_SESSION_OPEN_FILES

public static final java.lang.String SHUFFLE_MAX_SESSION_OPEN_FILES

See Also:

Constant Field Values

DEFAULT_SHUFFLE_MAX_SESSION_OPEN_FILES

public static final int DEFAULT_SHUFFLE_MAX_SESSION_OPEN_FILES

See Also:

Constant Field Values

ms

protected final org.apache.hadoop.metrics2.MetricsSystem ms

Constructor Detail

ShuffleHandler

public ShuffleHandler()

Method Detail

serializeMetaData

public static java.nio.ByteBuffer serializeMetaData(int port)
                                             throws java.io.IOException

Serialize the shuffle port into a ByteBuffer for use later on.

Parameters:

port - the port to be sent to the ApplciationMaster

Returns:

the serialized form of the port.

Throws:

java.io.IOException - on failure

deserializeMetaData

public static int deserializeMetaData(java.nio.ByteBuffer meta)
                               throws java.io.IOException

A helper function to deserialize the metadata returned by ShuffleHandler.

Parameters:

meta - the metadata returned by the ShuffleHandler

Returns:

the port the Shuffle Handler is listening on to serve shuffle data.

Throws:

java.io.IOException - on failure

serializeServiceData

public static java.nio.ByteBuffer serializeServiceData(org.apache.hadoop.security.token.Token<org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier> jobToken)
                                                throws java.io.IOException

A helper function to serialize the JobTokenIdentifier to be sent to the ShuffleHandler as ServiceData.

Parameters:

jobToken - the job token to be used for authentication of shuffle data requests.

Returns:

the serialized version of the jobToken.

Throws:

java.io.IOException - on failure

deserializeServiceData

public static org.apache.hadoop.security.token.Token<org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier> deserializeServiceData(java.nio.ByteBuffer secret)
                                                                                                                                    throws java.io.IOException

Throws:

java.io.IOException

initializeApplication

public void initializeApplication(org.apache.hadoop.yarn.server.api.ApplicationInitializationContext context)

Specified by:

initializeApplication in class org.apache.hadoop.yarn.server.api.AuxiliaryService

stopApplication

public void stopApplication(org.apache.hadoop.yarn.server.api.ApplicationTerminationContext context)

Specified by:

stopApplication in class org.apache.hadoop.yarn.server.api.AuxiliaryService

serviceInit

protected void serviceInit(org.apache.hadoop.conf.Configuration conf)
                    throws java.lang.Exception

Overrides:

serviceInit in class org.apache.hadoop.service.AbstractService

Throws:

java.lang.Exception

createHandlerContext

protected ShuffleChannelHandlerContext createHandlerContext()

serviceStart

protected void serviceStart()
                     throws java.lang.Exception

Overrides:

serviceStart in class org.apache.hadoop.service.AbstractService

Throws:

java.lang.Exception

serviceStop

protected void serviceStop()
                    throws java.lang.Exception

Overrides:

serviceStop in class org.apache.hadoop.service.AbstractService

Throws:

java.lang.Exception

getMetaData

public java.nio.ByteBuffer getMetaData()

Specified by:

getMetaData in class org.apache.hadoop.yarn.server.api.AuxiliaryService

getCurrentVersion

protected org.apache.hadoop.yarn.server.records.Version getCurrentVersion()