Package org.apache.hadoop.mapred

Class ShuffleHandler

java.lang.Object

org.apache.hadoop.service.AbstractService

org.apache.hadoop.yarn.server.api.AuxiliaryService

org.apache.hadoop.mapred.ShuffleHandler

All Implemented Interfaces:

Closeable, AutoCloseable, org.apache.hadoop.service.Service

public class ShuffleHandler
extends org.apache.hadoop.yarn.server.api.AuxiliaryService

Nested Class Summary

Nested classes/interfaces inherited from interface org.apache.hadoop.service.Service

org.apache.hadoop.service.Service.STATE

Field Summary

Fields

Modifier and Type	Field	Description
protected final io.netty.channel.group.ChannelGroup	allChannels
static final org.slf4j.Logger	AUDITLOG
static final String	CONCURRENCY_LEVEL
static final String	CONNECTION_CLOSE
protected static final org.apache.hadoop.yarn.server.records.Version	CURRENT_VERSION_INFO
static final int	DEFAULT_CONCURRENCY_LEVEL
static final int	DEFAULT_EXPIRE_AFTER_ACCESS_MINUTES
static final int	DEFAULT_MAX_SHUFFLE_CONNECTIONS
static final int	DEFAULT_MAX_SHUFFLE_THREADS
static final int	DEFAULT_MAX_WEIGHT
static final int	DEFAULT_SHUFFLE_BUFFER_SIZE
static final boolean	DEFAULT_SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED
static final int	DEFAULT_SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT
static final int	DEFAULT_SHUFFLE_LISTEN_QUEUE_SIZE
static final boolean	DEFAULT_SHUFFLE_MANAGE_OS_CACHE
static final int	DEFAULT_SHUFFLE_MAPOUTPUT_META_INFO_CACHE_SIZE
static final int	DEFAULT_SHUFFLE_MAX_SESSION_OPEN_FILES
static final int	DEFAULT_SHUFFLE_PORT
static final int	DEFAULT_SHUFFLE_READAHEAD_BYTES
static final boolean	DEFAULT_SHUFFLE_TRANSFERTO_ALLOWED
static final int	DEFAULT_SUFFLE_SSL_FILE_BUFFER_SIZE
static final String	EXPIRE_AFTER_ACCESS_MINUTES
static final long	FETCH_RETRY_DELAY
static final Pattern	IGNORABLE_ERROR_MESSAGE
static final org.slf4j.Logger	LOG
static final String	MAPREDUCE_SHUFFLE_SERVICEID
static final String	MAX_SHUFFLE_CONNECTIONS
static final String	MAX_SHUFFLE_THREADS
static final String	MAX_WEIGHT
protected final org.apache.hadoop.metrics2.MetricsSystem	ms
static final String	RETRY_AFTER_HEADER
protected org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager	secretManager
static final String	SHUFFLE_BUFFER_SIZE
static final String	SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED
static final String	SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT
static final String	SHUFFLE_LISTEN_QUEUE_SIZE
static final String	SHUFFLE_MANAGE_OS_CACHE
static final String	SHUFFLE_MAPOUTPUT_META_INFO_CACHE_SIZE
static final String	SHUFFLE_MAX_SESSION_OPEN_FILES
static final String	SHUFFLE_PORT_CONFIG_KEY
static final String	SHUFFLE_READAHEAD_BYTES
static final String	SHUFFLE_TRANSFERTO_ALLOWED
static final String	SUFFLE_SSL_FILE_BUFFER_SIZE_KEY
static final io.netty.handler.codec.http.HttpResponseStatus	TOO_MANY_REQ_STATUS
protected Map<String,String>	userRsrc
static final boolean	WINDOWS_DEFAULT_SHUFFLE_TRANSFERTO_ALLOWED

Constructor Summary

Constructors

Constructor	Description
ShuffleHandler()

Method Summary

Modifier and Type	Method	Description
protected ShuffleChannelHandlerContext	createHandlerContext()
static int	deserializeMetaData(ByteBuffer meta)	A helper function to deserialize the metadata returned by ShuffleHandler.
static org.apache.hadoop.security.token.Token<org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier>	deserializeServiceData(ByteBuffer secret)
protected org.apache.hadoop.yarn.server.records.Version	getCurrentVersion()
ByteBuffer	getMetaData()
void	initializeApplication(org.apache.hadoop.yarn.server.api.ApplicationInitializationContext context)
static ByteBuffer	serializeMetaData(int port)	Serialize the shuffle port into a ByteBuffer for use later on.
static ByteBuffer	serializeServiceData(org.apache.hadoop.security.token.Token<org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier> jobToken)	A helper function to serialize the JobTokenIdentifier to be sent to the ShuffleHandler as ServiceData.
protected void	serviceInit(org.apache.hadoop.conf.Configuration conf)
protected void	serviceStart()
protected void	serviceStop()
void	stopApplication(org.apache.hadoop.yarn.server.api.ApplicationTerminationContext context)

Methods inherited from class org.apache.hadoop.yarn.server.api.AuxiliaryService

getAuxiliaryLocalPathHandler, getRecoveryPath, initializeContainer, setAuxiliaryLocalPathHandler, setRecoveryPath, stopContainer

Methods inherited from class org.apache.hadoop.service.AbstractService

close, getBlockers, getConfig, getFailureCause, getFailureState, getLifecycleHistory, getName, getServiceState, getStartTime, init, isInState, noteFailure, putBlocker, registerGlobalListener, registerServiceListener, removeBlocker, setConfig, start, stop, toString, unregisterGlobalListener, unregisterServiceListener, waitForServiceToStop

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, wait, wait, wait

Field Details

LOG

public static final org.slf4j.Logger LOG

AUDITLOG

public static final org.slf4j.Logger AUDITLOG

SHUFFLE_MANAGE_OS_CACHE

public static final String SHUFFLE_MANAGE_OS_CACHE

See Also:

• Constant Field Values

DEFAULT_SHUFFLE_MANAGE_OS_CACHE

public static final boolean DEFAULT_SHUFFLE_MANAGE_OS_CACHE

See Also:

• Constant Field Values

SHUFFLE_READAHEAD_BYTES

public static final String SHUFFLE_READAHEAD_BYTES

See Also:

• Constant Field Values

DEFAULT_SHUFFLE_READAHEAD_BYTES

public static final int DEFAULT_SHUFFLE_READAHEAD_BYTES

See Also:

• Constant Field Values

MAX_WEIGHT

public static final String MAX_WEIGHT

See Also:

• Constant Field Values

DEFAULT_MAX_WEIGHT

public static final int DEFAULT_MAX_WEIGHT

See Also:

• Constant Field Values

EXPIRE_AFTER_ACCESS_MINUTES

public static final String EXPIRE_AFTER_ACCESS_MINUTES

See Also:

• Constant Field Values

DEFAULT_EXPIRE_AFTER_ACCESS_MINUTES

public static final int DEFAULT_EXPIRE_AFTER_ACCESS_MINUTES

See Also:

• Constant Field Values

CONCURRENCY_LEVEL

public static final String CONCURRENCY_LEVEL

See Also:

• Constant Field Values

DEFAULT_CONCURRENCY_LEVEL

public static final int DEFAULT_CONCURRENCY_LEVEL

See Also:

• Constant Field Values

IGNORABLE_ERROR_MESSAGE

public static final Pattern IGNORABLE_ERROR_MESSAGE

CURRENT_VERSION_INFO

protected static final org.apache.hadoop.yarn.server.records.Version CURRENT_VERSION_INFO

TOO_MANY_REQ_STATUS

public static final io.netty.handler.codec.http.HttpResponseStatus TOO_MANY_REQ_STATUS

FETCH_RETRY_DELAY

public static final long FETCH_RETRY_DELAY

See Also:

• Constant Field Values

RETRY_AFTER_HEADER

public static final String RETRY_AFTER_HEADER

See Also:

• Constant Field Values

allChannels

protected final io.netty.channel.group.ChannelGroup allChannels

secretManager

protected org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager secretManager

userRsrc

protected Map<String,String> userRsrc

MAPREDUCE_SHUFFLE_SERVICEID

public static final String MAPREDUCE_SHUFFLE_SERVICEID

See Also:

• Constant Field Values

SHUFFLE_PORT_CONFIG_KEY

public static final String SHUFFLE_PORT_CONFIG_KEY

See Also:

• Constant Field Values

DEFAULT_SHUFFLE_PORT

public static final int DEFAULT_SHUFFLE_PORT

See Also:

• Constant Field Values

SHUFFLE_LISTEN_QUEUE_SIZE

public static final String SHUFFLE_LISTEN_QUEUE_SIZE

See Also:

• Constant Field Values

DEFAULT_SHUFFLE_LISTEN_QUEUE_SIZE

public static final int DEFAULT_SHUFFLE_LISTEN_QUEUE_SIZE

See Also:

• Constant Field Values

SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED

public static final String SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED

See Also:

• Constant Field Values

DEFAULT_SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED

public static final boolean DEFAULT_SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED

See Also:

• Constant Field Values

SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT

public static final String SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT

See Also:

• Constant Field Values

DEFAULT_SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT

public static final int DEFAULT_SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT

See Also:

• Constant Field Values

SHUFFLE_MAPOUTPUT_META_INFO_CACHE_SIZE

public static final String SHUFFLE_MAPOUTPUT_META_INFO_CACHE_SIZE

See Also:

• Constant Field Values

DEFAULT_SHUFFLE_MAPOUTPUT_META_INFO_CACHE_SIZE

public static final int DEFAULT_SHUFFLE_MAPOUTPUT_META_INFO_CACHE_SIZE

See Also:

• Constant Field Values

CONNECTION_CLOSE

public static final String CONNECTION_CLOSE

See Also:

• Constant Field Values

SUFFLE_SSL_FILE_BUFFER_SIZE_KEY

public static final String SUFFLE_SSL_FILE_BUFFER_SIZE_KEY

See Also:

• Constant Field Values

DEFAULT_SUFFLE_SSL_FILE_BUFFER_SIZE

public static final int DEFAULT_SUFFLE_SSL_FILE_BUFFER_SIZE

See Also:

• Constant Field Values

MAX_SHUFFLE_CONNECTIONS

public static final String MAX_SHUFFLE_CONNECTIONS

See Also:

• Constant Field Values

DEFAULT_MAX_SHUFFLE_CONNECTIONS

public static final int DEFAULT_MAX_SHUFFLE_CONNECTIONS

See Also:

• Constant Field Values

MAX_SHUFFLE_THREADS

public static final String MAX_SHUFFLE_THREADS

See Also:

• Constant Field Values

DEFAULT_MAX_SHUFFLE_THREADS

public static final int DEFAULT_MAX_SHUFFLE_THREADS

See Also:

• Constant Field Values

SHUFFLE_BUFFER_SIZE

public static final String SHUFFLE_BUFFER_SIZE

See Also:

• Constant Field Values

DEFAULT_SHUFFLE_BUFFER_SIZE

public static final int DEFAULT_SHUFFLE_BUFFER_SIZE

See Also:

• Constant Field Values

SHUFFLE_TRANSFERTO_ALLOWED

public static final String SHUFFLE_TRANSFERTO_ALLOWED

See Also:

• Constant Field Values

DEFAULT_SHUFFLE_TRANSFERTO_ALLOWED

public static final boolean DEFAULT_SHUFFLE_TRANSFERTO_ALLOWED

See Also:

• Constant Field Values

WINDOWS_DEFAULT_SHUFFLE_TRANSFERTO_ALLOWED

public static final boolean WINDOWS_DEFAULT_SHUFFLE_TRANSFERTO_ALLOWED

See Also:

• Constant Field Values

SHUFFLE_MAX_SESSION_OPEN_FILES

public static final String SHUFFLE_MAX_SESSION_OPEN_FILES

See Also:

• Constant Field Values

DEFAULT_SHUFFLE_MAX_SESSION_OPEN_FILES

public static final int DEFAULT_SHUFFLE_MAX_SESSION_OPEN_FILES

See Also:

• Constant Field Values

ms

protected final org.apache.hadoop.metrics2.MetricsSystem ms

Constructor Details

ShuffleHandler

public ShuffleHandler()

Method Details

serializeMetaData

public static ByteBuffer serializeMetaData(int port)
                                    throws IOException

Serialize the shuffle port into a ByteBuffer for use later on.

Parameters:

port - the port to be sent to the ApplciationMaster

Returns:

the serialized form of the port.

Throws:

IOException - on failure

deserializeMetaData

public static int deserializeMetaData(ByteBuffer meta)
                               throws IOException

A helper function to deserialize the metadata returned by ShuffleHandler.

Parameters:

meta - the metadata returned by the ShuffleHandler

Returns:

the port the Shuffle Handler is listening on to serve shuffle data.

Throws:

IOException - on failure

serializeServiceData

public static ByteBuffer serializeServiceData(org.apache.hadoop.security.token.Token<org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier> jobToken)
                                       throws IOException

A helper function to serialize the JobTokenIdentifier to be sent to the ShuffleHandler as ServiceData.

Parameters:

jobToken - the job token to be used for authentication of shuffle data requests.

Returns:

the serialized version of the jobToken.

Throws:

IOException - on failure

deserializeServiceData

public static org.apache.hadoop.security.token.Token<org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier> deserializeServiceData(ByteBuffer secret)
                                                                                                                                    throws IOException

Throws:

IOException

initializeApplication

public void initializeApplication(org.apache.hadoop.yarn.server.api.ApplicationInitializationContext context)

Specified by:

initializeApplication in class org.apache.hadoop.yarn.server.api.AuxiliaryService

stopApplication

public void stopApplication(org.apache.hadoop.yarn.server.api.ApplicationTerminationContext context)

Specified by:

stopApplication in class org.apache.hadoop.yarn.server.api.AuxiliaryService

serviceInit

protected void serviceInit(org.apache.hadoop.conf.Configuration conf)
                    throws Exception

Overrides:

serviceInit in class org.apache.hadoop.service.AbstractService

Throws:

Exception

createHandlerContext

protected ShuffleChannelHandlerContext createHandlerContext()

serviceStart

protected void serviceStart()
                     throws Exception

Overrides:

serviceStart in class org.apache.hadoop.service.AbstractService

Throws:

Exception

serviceStop

protected void serviceStop()
                    throws Exception

Overrides:

serviceStop in class org.apache.hadoop.service.AbstractService

Throws:

Exception

getMetaData

public ByteBuffer getMetaData()

Specified by:

getMetaData in class org.apache.hadoop.yarn.server.api.AuxiliaryService

getCurrentVersion

protected org.apache.hadoop.yarn.server.records.Version getCurrentVersion()