org.apache.hadoop.mapred
Class ShuffleHandler
java.lang.Object
org.apache.hadoop.service.AbstractService
org.apache.hadoop.yarn.server.api.AuxiliaryService
org.apache.hadoop.mapred.ShuffleHandler
- All Implemented Interfaces:
- Closeable, org.apache.hadoop.service.Service
public class ShuffleHandler
- extends org.apache.hadoop.yarn.server.api.AuxiliaryService
Nested classes/interfaces inherited from interface org.apache.hadoop.service.Service |
org.apache.hadoop.service.Service.STATE |
Method Summary |
static int |
deserializeMetaData(ByteBuffer meta)
A helper function to deserialize the metadata returned by ShuffleHandler. |
ByteBuffer |
getMetaData()
|
protected org.apache.hadoop.mapred.ShuffleHandler.Shuffle |
getShuffle(org.apache.hadoop.conf.Configuration conf)
|
void |
initializeApplication(org.apache.hadoop.yarn.server.api.ApplicationInitializationContext context)
|
static ByteBuffer |
serializeMetaData(int port)
Serialize the shuffle port into a ByteBuffer for use later on. |
static ByteBuffer |
serializeServiceData(org.apache.hadoop.security.token.Token<org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier> jobToken)
A helper function to serialize the JobTokenIdentifier to be sent to the
ShuffleHandler as ServiceData. |
protected void |
serviceInit(org.apache.hadoop.conf.Configuration conf)
|
protected void |
serviceStart()
|
protected void |
serviceStop()
|
void |
stopApplication(org.apache.hadoop.yarn.server.api.ApplicationTerminationContext context)
|
Methods inherited from class org.apache.hadoop.yarn.server.api.AuxiliaryService |
getMetaData, initializeContainer, stopContainer |
Methods inherited from class org.apache.hadoop.service.AbstractService |
close, getBlockers, getConfig, getFailureCause, getFailureState, getLifecycleHistory, getName, getServiceState, getStartTime, init, isInState, noteFailure, putBlocker, registerGlobalListener, registerServiceListener, removeBlocker, setConfig, start, stop, toString, unregisterGlobalListener, unregisterServiceListener, waitForServiceToStop |
SHUFFLE_MANAGE_OS_CACHE
public static final String SHUFFLE_MANAGE_OS_CACHE
- See Also:
- Constant Field Values
DEFAULT_SHUFFLE_MANAGE_OS_CACHE
public static final boolean DEFAULT_SHUFFLE_MANAGE_OS_CACHE
- See Also:
- Constant Field Values
SHUFFLE_READAHEAD_BYTES
public static final String SHUFFLE_READAHEAD_BYTES
- See Also:
- Constant Field Values
DEFAULT_SHUFFLE_READAHEAD_BYTES
public static final int DEFAULT_SHUFFLE_READAHEAD_BYTES
- See Also:
- Constant Field Values
pipelineFact
protected org.apache.hadoop.mapred.ShuffleHandler.HttpPipelineFactory pipelineFact
MAPREDUCE_SHUFFLE_SERVICEID
public static final String MAPREDUCE_SHUFFLE_SERVICEID
- See Also:
- Constant Field Values
SHUFFLE_PORT_CONFIG_KEY
public static final String SHUFFLE_PORT_CONFIG_KEY
- See Also:
- Constant Field Values
DEFAULT_SHUFFLE_PORT
public static final int DEFAULT_SHUFFLE_PORT
- See Also:
- Constant Field Values
SUFFLE_SSL_FILE_BUFFER_SIZE_KEY
public static final String SUFFLE_SSL_FILE_BUFFER_SIZE_KEY
- See Also:
- Constant Field Values
DEFAULT_SUFFLE_SSL_FILE_BUFFER_SIZE
public static final int DEFAULT_SUFFLE_SSL_FILE_BUFFER_SIZE
- See Also:
- Constant Field Values
MAX_SHUFFLE_CONNECTIONS
public static final String MAX_SHUFFLE_CONNECTIONS
- See Also:
- Constant Field Values
DEFAULT_MAX_SHUFFLE_CONNECTIONS
public static final int DEFAULT_MAX_SHUFFLE_CONNECTIONS
- See Also:
- Constant Field Values
MAX_SHUFFLE_THREADS
public static final String MAX_SHUFFLE_THREADS
- See Also:
- Constant Field Values
DEFAULT_MAX_SHUFFLE_THREADS
public static final int DEFAULT_MAX_SHUFFLE_THREADS
- See Also:
- Constant Field Values
ShuffleHandler
public ShuffleHandler()
serializeMetaData
public static ByteBuffer serializeMetaData(int port)
throws IOException
- Serialize the shuffle port into a ByteBuffer for use later on.
- Parameters:
port
- the port to be sent to the ApplciationMaster
- Returns:
- the serialized form of the port.
- Throws:
IOException
deserializeMetaData
public static int deserializeMetaData(ByteBuffer meta)
throws IOException
- A helper function to deserialize the metadata returned by ShuffleHandler.
- Parameters:
meta
- the metadata returned by the ShuffleHandler
- Returns:
- the port the Shuffle Handler is listening on to serve shuffle data.
- Throws:
IOException
serializeServiceData
public static ByteBuffer serializeServiceData(org.apache.hadoop.security.token.Token<org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier> jobToken)
throws IOException
- A helper function to serialize the JobTokenIdentifier to be sent to the
ShuffleHandler as ServiceData.
- Parameters:
jobToken
- the job token to be used for authentication of
shuffle data requests.
- Returns:
- the serialized version of the jobToken.
- Throws:
IOException
initializeApplication
public void initializeApplication(org.apache.hadoop.yarn.server.api.ApplicationInitializationContext context)
- Specified by:
initializeApplication
in class org.apache.hadoop.yarn.server.api.AuxiliaryService
stopApplication
public void stopApplication(org.apache.hadoop.yarn.server.api.ApplicationTerminationContext context)
- Specified by:
stopApplication
in class org.apache.hadoop.yarn.server.api.AuxiliaryService
serviceInit
protected void serviceInit(org.apache.hadoop.conf.Configuration conf)
throws Exception
- Overrides:
serviceInit
in class org.apache.hadoop.service.AbstractService
- Throws:
Exception
serviceStart
protected void serviceStart()
throws Exception
- Overrides:
serviceStart
in class org.apache.hadoop.service.AbstractService
- Throws:
Exception
serviceStop
protected void serviceStop()
throws Exception
- Overrides:
serviceStop
in class org.apache.hadoop.service.AbstractService
- Throws:
Exception
getMetaData
public ByteBuffer getMetaData()
- Specified by:
getMetaData
in class org.apache.hadoop.yarn.server.api.AuxiliaryService
getShuffle
protected org.apache.hadoop.mapred.ShuffleHandler.Shuffle getShuffle(org.apache.hadoop.conf.Configuration conf)
Copyright © 2014 Apache Software Foundation. All Rights Reserved.