org.apache.hadoop.mapred
Class ShuffleHandler

java.lang.Object
  extended by org.apache.hadoop.service.AbstractService
      extended by org.apache.hadoop.yarn.server.api.AuxiliaryService
          extended by org.apache.hadoop.mapred.ShuffleHandler
All Implemented Interfaces:
Closeable, org.apache.hadoop.service.Service

public class ShuffleHandler
extends org.apache.hadoop.yarn.server.api.AuxiliaryService


Nested Class Summary
 
Nested classes/interfaces inherited from interface org.apache.hadoop.service.Service
org.apache.hadoop.service.Service.STATE
 
Field Summary
static int DEFAULT_MAX_SHUFFLE_CONNECTIONS
           
static int DEFAULT_MAX_SHUFFLE_THREADS
           
static boolean DEFAULT_SHUFFLE_MANAGE_OS_CACHE
           
static int DEFAULT_SHUFFLE_PORT
           
static int DEFAULT_SHUFFLE_READAHEAD_BYTES
           
static int DEFAULT_SUFFLE_SSL_FILE_BUFFER_SIZE
           
static String MAPREDUCE_SHUFFLE_SERVICEID
           
static String MAX_SHUFFLE_CONNECTIONS
           
static String MAX_SHUFFLE_THREADS
           
protected  org.apache.hadoop.mapred.ShuffleHandler.HttpPipelineFactory pipelineFact
           
static String SHUFFLE_MANAGE_OS_CACHE
           
static String SHUFFLE_PORT_CONFIG_KEY
           
static String SHUFFLE_READAHEAD_BYTES
           
static String SUFFLE_SSL_FILE_BUFFER_SIZE_KEY
           
 
Constructor Summary
ShuffleHandler()
           
 
Method Summary
static int deserializeMetaData(ByteBuffer meta)
          A helper function to deserialize the metadata returned by ShuffleHandler.
 ByteBuffer getMetaData()
           
protected  org.apache.hadoop.mapred.ShuffleHandler.Shuffle getShuffle(org.apache.hadoop.conf.Configuration conf)
           
 void initializeApplication(org.apache.hadoop.yarn.server.api.ApplicationInitializationContext context)
           
static ByteBuffer serializeMetaData(int port)
          Serialize the shuffle port into a ByteBuffer for use later on.
static ByteBuffer serializeServiceData(org.apache.hadoop.security.token.Token<org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier> jobToken)
          A helper function to serialize the JobTokenIdentifier to be sent to the ShuffleHandler as ServiceData.
protected  void serviceInit(org.apache.hadoop.conf.Configuration conf)
           
protected  void serviceStart()
           
protected  void serviceStop()
           
 void stopApplication(org.apache.hadoop.yarn.server.api.ApplicationTerminationContext context)
           
 
Methods inherited from class org.apache.hadoop.yarn.server.api.AuxiliaryService
getMetaData, initializeContainer, stopContainer
 
Methods inherited from class org.apache.hadoop.service.AbstractService
close, getBlockers, getConfig, getFailureCause, getFailureState, getLifecycleHistory, getName, getServiceState, getStartTime, init, isInState, noteFailure, putBlocker, registerGlobalListener, registerServiceListener, removeBlocker, setConfig, start, stop, toString, unregisterGlobalListener, unregisterServiceListener, waitForServiceToStop
 
Methods inherited from class java.lang.Object
clone, equals, finalize, getClass, hashCode, notify, notifyAll, wait, wait, wait
 

Field Detail

SHUFFLE_MANAGE_OS_CACHE

public static final String SHUFFLE_MANAGE_OS_CACHE
See Also:
Constant Field Values

DEFAULT_SHUFFLE_MANAGE_OS_CACHE

public static final boolean DEFAULT_SHUFFLE_MANAGE_OS_CACHE
See Also:
Constant Field Values

SHUFFLE_READAHEAD_BYTES

public static final String SHUFFLE_READAHEAD_BYTES
See Also:
Constant Field Values

DEFAULT_SHUFFLE_READAHEAD_BYTES

public static final int DEFAULT_SHUFFLE_READAHEAD_BYTES
See Also:
Constant Field Values

pipelineFact

protected org.apache.hadoop.mapred.ShuffleHandler.HttpPipelineFactory pipelineFact

MAPREDUCE_SHUFFLE_SERVICEID

public static final String MAPREDUCE_SHUFFLE_SERVICEID
See Also:
Constant Field Values

SHUFFLE_PORT_CONFIG_KEY

public static final String SHUFFLE_PORT_CONFIG_KEY
See Also:
Constant Field Values

DEFAULT_SHUFFLE_PORT

public static final int DEFAULT_SHUFFLE_PORT
See Also:
Constant Field Values

SUFFLE_SSL_FILE_BUFFER_SIZE_KEY

public static final String SUFFLE_SSL_FILE_BUFFER_SIZE_KEY
See Also:
Constant Field Values

DEFAULT_SUFFLE_SSL_FILE_BUFFER_SIZE

public static final int DEFAULT_SUFFLE_SSL_FILE_BUFFER_SIZE
See Also:
Constant Field Values

MAX_SHUFFLE_CONNECTIONS

public static final String MAX_SHUFFLE_CONNECTIONS
See Also:
Constant Field Values

DEFAULT_MAX_SHUFFLE_CONNECTIONS

public static final int DEFAULT_MAX_SHUFFLE_CONNECTIONS
See Also:
Constant Field Values

MAX_SHUFFLE_THREADS

public static final String MAX_SHUFFLE_THREADS
See Also:
Constant Field Values

DEFAULT_MAX_SHUFFLE_THREADS

public static final int DEFAULT_MAX_SHUFFLE_THREADS
See Also:
Constant Field Values
Constructor Detail

ShuffleHandler

public ShuffleHandler()
Method Detail

serializeMetaData

public static ByteBuffer serializeMetaData(int port)
                                    throws IOException
Serialize the shuffle port into a ByteBuffer for use later on.

Parameters:
port - the port to be sent to the ApplciationMaster
Returns:
the serialized form of the port.
Throws:
IOException

deserializeMetaData

public static int deserializeMetaData(ByteBuffer meta)
                               throws IOException
A helper function to deserialize the metadata returned by ShuffleHandler.

Parameters:
meta - the metadata returned by the ShuffleHandler
Returns:
the port the Shuffle Handler is listening on to serve shuffle data.
Throws:
IOException

serializeServiceData

public static ByteBuffer serializeServiceData(org.apache.hadoop.security.token.Token<org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier> jobToken)
                                       throws IOException
A helper function to serialize the JobTokenIdentifier to be sent to the ShuffleHandler as ServiceData.

Parameters:
jobToken - the job token to be used for authentication of shuffle data requests.
Returns:
the serialized version of the jobToken.
Throws:
IOException

initializeApplication

public void initializeApplication(org.apache.hadoop.yarn.server.api.ApplicationInitializationContext context)
Specified by:
initializeApplication in class org.apache.hadoop.yarn.server.api.AuxiliaryService

stopApplication

public void stopApplication(org.apache.hadoop.yarn.server.api.ApplicationTerminationContext context)
Specified by:
stopApplication in class org.apache.hadoop.yarn.server.api.AuxiliaryService

serviceInit

protected void serviceInit(org.apache.hadoop.conf.Configuration conf)
                    throws Exception
Overrides:
serviceInit in class org.apache.hadoop.service.AbstractService
Throws:
Exception

serviceStart

protected void serviceStart()
                     throws Exception
Overrides:
serviceStart in class org.apache.hadoop.service.AbstractService
Throws:
Exception

serviceStop

protected void serviceStop()
                    throws Exception
Overrides:
serviceStop in class org.apache.hadoop.service.AbstractService
Throws:
Exception

getMetaData

public ByteBuffer getMetaData()
Specified by:
getMetaData in class org.apache.hadoop.yarn.server.api.AuxiliaryService

getShuffle

protected org.apache.hadoop.mapred.ShuffleHandler.Shuffle getShuffle(org.apache.hadoop.conf.Configuration conf)


Copyright © 2014 Apache Software Foundation. All Rights Reserved.