public class FederationInterceptor extends AbstractRequestInterceptor
| Modifier and Type | Field and Description |
|---|---|
static String |
NMSS_CLASS_PREFIX |
static String |
NMSS_REG_REQUEST_KEY |
static String |
NMSS_REG_RESPONSE_KEY |
static String |
NMSS_SECONDARY_SC_PREFIX
When AMRMProxy HA is enabled, secondary AMRMTokens will be stored in Yarn
Registry.
|
static String |
STRING_TO_BYTE_FORMAT |
| Constructor and Description |
|---|
FederationInterceptor()
Creates an instance of the FederationInterceptor class.
|
| Modifier and Type | Method and Description |
|---|---|
org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse |
allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest request)
Sends the heart beats to the home RM and the secondary sub-cluster RMs that
are being used by the application.
|
protected void |
cleanupRegistry()
Only for unit test cleanup.
|
protected org.apache.hadoop.yarn.server.AMHeartbeatRequestHandler |
createHomeHeartbeartHandler(org.apache.hadoop.conf.Configuration conf,
org.apache.hadoop.yarn.api.records.ApplicationId appId,
org.apache.hadoop.yarn.server.AMRMClientRelayer rmProxyRelayer) |
protected <T> T |
createHomeRMProxy(AMRMProxyApplicationContext appContext,
Class<T> protocol,
org.apache.hadoop.security.UserGroupInformation user)
Create a proxy instance that is used to connect to the Home resource
manager.
|
protected org.apache.hadoop.yarn.server.uam.UnmanagedAMPoolManager |
createUnmanagedAMPoolManager(ExecutorService threadPool)
Create the UAM pool manager for secondary sub-clsuters.
|
org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterResponse |
finishApplicationMaster(org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest request)
Sends the finish application master request to all the resource managers
used by the application.
|
protected org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse |
generateBaseAllocationResponse()
Prepare the base allocation response.
|
Map<org.apache.hadoop.yarn.server.federation.store.records.SubClusterId,List<org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse>> |
getAsyncResponseSink() |
protected org.apache.hadoop.yarn.api.records.ApplicationAttemptId |
getAttemptId() |
protected org.apache.hadoop.yarn.server.AMHeartbeatRequestHandler |
getHomeHeartbeartHandler() |
protected org.apache.hadoop.yarn.server.federation.utils.FederationRegistryClient |
getRegistryClient() |
protected Set<org.apache.hadoop.yarn.server.federation.store.records.SubClusterId> |
getTimedOutSCs(boolean verbose) |
protected Map<org.apache.hadoop.yarn.server.federation.store.records.SubClusterId,Future<?>> |
getUamRegisterFutures() |
protected org.apache.hadoop.yarn.server.uam.UnmanagedAMPoolManager |
getUnmanagedAMPool() |
protected int |
getUnmanagedAMPoolSize() |
void |
init(AMRMProxyApplicationContext appContext)
Initializes the instance using specified context.
|
static <T> boolean |
isNullOrEmpty(Collection<T> c)
Utility method to check if the specified Collection is null or empty.
|
static <T1,T2> boolean |
isNullOrEmpty(Map<T1,T2> c)
Utility method to check if the specified Collection is null or empty.
|
protected void |
mergeAllocateResponse(org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse homeResponse,
org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse otherResponse,
org.apache.hadoop.yarn.server.federation.store.records.SubClusterId otherRMAddress) |
protected void |
reAttachUAMAndMergeRegisterResponse(org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse homeResponse,
org.apache.hadoop.yarn.api.records.ApplicationId appId)
Try re-attach to all existing and running UAMs in secondary sub-clusters
launched by previous application attempts if any.
|
void |
recover(Map<String,byte[]> recoveredDataMap)
Recover
RequestInterceptor state from store. |
org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse |
registerApplicationMaster(org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest request)
Sends the application master's registration request to the home RM.
|
void |
setNextInterceptor(RequestInterceptor next)
Sets the
RequestInterceptor in the chain. |
void |
shutdown()
This is called when the application pipeline is being destroyed.
|
protected Map<org.apache.hadoop.yarn.server.federation.store.records.SubClusterId,List<org.apache.hadoop.yarn.api.records.ResourceRequest>> |
splitResourceRequests(List<org.apache.hadoop.yarn.api.records.ResourceRequest> askList)
Splits the specified request to send it to different sub clusters.
|
allocateForDistributedScheduling, getApplicationContext, getConf, getNextInterceptor, getNMStateStore, registerApplicationMasterForDistributedScheduling, setConfpublic static final String NMSS_CLASS_PREFIX
public static final String NMSS_REG_REQUEST_KEY
public static final String NMSS_REG_RESPONSE_KEY
public static final String NMSS_SECONDARY_SC_PREFIX
public static final String STRING_TO_BYTE_FORMAT
public FederationInterceptor()
public void init(AMRMProxyApplicationContext appContext)
init in interface RequestInterceptorinit in class AbstractRequestInterceptorappContext - AMRMProxy application contextpublic void recover(Map<String,byte[]> recoveredDataMap)
AbstractRequestInterceptorRequestInterceptor state from store.recover in interface RequestInterceptorrecover in class AbstractRequestInterceptorrecoveredDataMap - states for all intercepters recovered from NMSSpublic org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse registerApplicationMaster(org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest request)
throws org.apache.hadoop.yarn.exceptions.YarnException,
IOException
InvalidApplicationMasterRequestException. Furthermore, we present
to AM as if we are the RM that never fails over (except when AMRMProxy
restarts). When actual RM fails over, we always re-register automatically.
We did this because FederationInterceptor can receive concurrent register
requests from AM because of timeout between AM and AMRMProxy, which is
shorter than the timeout + failOver between FederationInterceptor
(AMRMProxy) and RM.
For the same reason, this method needs to be synchronized.org.apache.hadoop.yarn.exceptions.YarnExceptionIOExceptionpublic org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest request)
throws org.apache.hadoop.yarn.exceptions.YarnException,
IOException
org.apache.hadoop.yarn.exceptions.YarnExceptionIOExceptionpublic org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterResponse finishApplicationMaster(org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest request)
throws org.apache.hadoop.yarn.exceptions.YarnException,
IOException
org.apache.hadoop.yarn.exceptions.YarnExceptionIOExceptionpublic void setNextInterceptor(RequestInterceptor next)
AbstractRequestInterceptorRequestInterceptor in the chain.setNextInterceptor in interface RequestInterceptorsetNextInterceptor in class AbstractRequestInterceptornext - the next intercepter to setpublic void shutdown()
shutdown in interface RequestInterceptorshutdown in class AbstractRequestInterceptorprotected void cleanupRegistry()
protected org.apache.hadoop.yarn.server.federation.utils.FederationRegistryClient getRegistryClient()
protected org.apache.hadoop.yarn.api.records.ApplicationAttemptId getAttemptId()
protected org.apache.hadoop.yarn.server.AMHeartbeatRequestHandler getHomeHeartbeartHandler()
protected org.apache.hadoop.yarn.server.uam.UnmanagedAMPoolManager createUnmanagedAMPoolManager(ExecutorService threadPool)
threadPool - the thread pool to useprotected org.apache.hadoop.yarn.server.AMHeartbeatRequestHandler createHomeHeartbeartHandler(org.apache.hadoop.conf.Configuration conf,
org.apache.hadoop.yarn.api.records.ApplicationId appId,
org.apache.hadoop.yarn.server.AMRMClientRelayer rmProxyRelayer)
protected <T> T createHomeRMProxy(AMRMProxyApplicationContext appContext, Class<T> protocol, org.apache.hadoop.security.UserGroupInformation user)
T - the type of the proxyappContext - AMRMProxyApplicationContextprotocol - the protocol class for the proxyuser - the ugi for the proxyprotected void reAttachUAMAndMergeRegisterResponse(org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse homeResponse,
org.apache.hadoop.yarn.api.records.ApplicationId appId)
protected org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse generateBaseAllocationResponse()
protected void mergeAllocateResponse(org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse homeResponse,
org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse otherResponse,
org.apache.hadoop.yarn.server.federation.store.records.SubClusterId otherRMAddress)
protected Set<org.apache.hadoop.yarn.server.federation.store.records.SubClusterId> getTimedOutSCs(boolean verbose)
protected Map<org.apache.hadoop.yarn.server.federation.store.records.SubClusterId,List<org.apache.hadoop.yarn.api.records.ResourceRequest>> splitResourceRequests(List<org.apache.hadoop.yarn.api.records.ResourceRequest> askList) throws org.apache.hadoop.yarn.exceptions.YarnException
askList - the ask list to splitorg.apache.hadoop.yarn.exceptions.YarnException - if split failsprotected int getUnmanagedAMPoolSize()
protected org.apache.hadoop.yarn.server.uam.UnmanagedAMPoolManager getUnmanagedAMPool()
protected Map<org.apache.hadoop.yarn.server.federation.store.records.SubClusterId,Future<?>> getUamRegisterFutures()
public Map<org.apache.hadoop.yarn.server.federation.store.records.SubClusterId,List<org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse>> getAsyncResponseSink()
public static <T> boolean isNullOrEmpty(Collection<T> c)
T - element type of the collectionc - the collection objectpublic static <T1,T2> boolean isNullOrEmpty(Map<T1,T2> c)
T1 - key type of the mapT2 - value type of the mapc - the map objectCopyright © 2008–2023 Apache Software Foundation. All rights reserved.