Package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity

Class CapacitySchedulerConfiguration

java.lang.Object
org.apache.hadoop.conf.Configuration
org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSchedulerConfiguration
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration
All Implemented Interfaces:
Iterable<Map.Entry<String,String>>, org.apache.hadoop.io.Writable
public class CapacitySchedulerConfiguration extends ReservationSchedulerConfiguration

  • Field Details

    • PREFIX

      @Private public static final String PREFIX
      See Also:

    • DOT

      @Private public static final String DOT
      See Also:

    • MAXIMUM_APPLICATIONS_SUFFIX

      @Private public static final String MAXIMUM_APPLICATIONS_SUFFIX
      See Also:

    • MAXIMUM_SYSTEM_APPLICATIONS

      @Private public static final String MAXIMUM_SYSTEM_APPLICATIONS
      See Also:

    • MAXIMUM_AM_RESOURCE_SUFFIX

      @Private public static final String MAXIMUM_AM_RESOURCE_SUFFIX
      See Also:

    • MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT

      @Private public static final String MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT
      See Also:

    • QUEUES

      @Private public static final String QUEUES
      See Also:

    • CAPACITY

      @Private public static final String CAPACITY
      See Also:

    • MAXIMUM_CAPACITY

      @Private public static final String MAXIMUM_CAPACITY
      See Also:

    • USER_LIMIT

      @Private public static final String USER_LIMIT
      See Also:

    • USER_LIMIT_FACTOR

      @Private public static final String USER_LIMIT_FACTOR
      See Also:

    • USER_WEIGHT

      @Private public static final String USER_WEIGHT
      See Also:

    • USER_SETTINGS

      @Private public static final String USER_SETTINGS
      See Also:

    • USER_WEIGHT_REGEX

      @Private public static final String USER_WEIGHT_REGEX
      See Also:

    • USER_WEIGHT_PATTERN

      @Private public static final Pattern USER_WEIGHT_PATTERN

    • DEFAULT_USER_WEIGHT

      @Private public static final float DEFAULT_USER_WEIGHT
      See Also:

    • STATE

      @Private public static final String STATE
      See Also:

    • ACCESSIBLE_NODE_LABELS

      @Private public static final String ACCESSIBLE_NODE_LABELS
      See Also:

    • DEFAULT_NODE_LABEL_EXPRESSION

      @Private public static final String DEFAULT_NODE_LABEL_EXPRESSION
      See Also:

    • RESERVE_CONT_LOOK_ALL_NODES

      public static final String RESERVE_CONT_LOOK_ALL_NODES
      See Also:

    • DEFAULT_RESERVE_CONT_LOOK_ALL_NODES

      @Private public static final boolean DEFAULT_RESERVE_CONT_LOOK_ALL_NODES
      See Also:

    • SKIP_ALLOCATE_ON_NODES_WITH_RESERVED_CONTAINERS

      public static final String SKIP_ALLOCATE_ON_NODES_WITH_RESERVED_CONTAINERS
      See Also:

    • DEFAULT_SKIP_ALLOCATE_ON_NODES_WITH_RESERVED_CONTAINERS

      @Private public static final boolean DEFAULT_SKIP_ALLOCATE_ON_NODES_WITH_RESERVED_CONTAINERS
      See Also:

    • MAXIMUM_ALLOCATION

      @Private public static final String MAXIMUM_ALLOCATION
      See Also:

    • MAXIMUM_ALLOCATION_MB

      @Private public static final String MAXIMUM_ALLOCATION_MB
      See Also:

    • MAXIMUM_ALLOCATION_VCORES

      @Private public static final String MAXIMUM_ALLOCATION_VCORES
      See Also:

    • ORDERING_POLICY

      public static final String ORDERING_POLICY
      Ordering policy of queues
      See Also:

    • FIFO_APP_ORDERING_POLICY

      public static final String FIFO_APP_ORDERING_POLICY
      See Also:

    • FAIR_APP_ORDERING_POLICY

      public static final String FAIR_APP_ORDERING_POLICY
      See Also:

    • FIFO_WITH_PARTITIONS_APP_ORDERING_POLICY

      public static final String FIFO_WITH_PARTITIONS_APP_ORDERING_POLICY
      See Also:

    • FIFO_FOR_PENDING_APPS

      public static final String FIFO_FOR_PENDING_APPS
      See Also:

    • DEFAULT_APP_ORDERING_POLICY

      public static final String DEFAULT_APP_ORDERING_POLICY
      See Also:

    • DEFAULT_MAXIMUM_SYSTEM_APPLICATIIONS

      @Private public static final int DEFAULT_MAXIMUM_SYSTEM_APPLICATIIONS
      See Also:

    • DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT

      @Private public static final float DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT
      See Also:

    • UNDEFINED

      @Private public static final float UNDEFINED
      See Also:

    • MINIMUM_CAPACITY_VALUE

      @Private public static final float MINIMUM_CAPACITY_VALUE
      See Also:

    • MAXIMUM_CAPACITY_VALUE

      @Private public static final float MAXIMUM_CAPACITY_VALUE
      See Also:

    • DEFAULT_MAXIMUM_CAPACITY_VALUE

      @Private public static final float DEFAULT_MAXIMUM_CAPACITY_VALUE
      See Also:

    • DEFAULT_USER_LIMIT

      @Private public static final int DEFAULT_USER_LIMIT
      See Also:

    • DEFAULT_USER_LIMIT_FACTOR

      @Private public static final float DEFAULT_USER_LIMIT_FACTOR
      See Also:

    • ALL_ACL

      @Private public static final String ALL_ACL
      See Also:

    • NONE_ACL

      @Private public static final String NONE_ACL
      See Also:

    • ENABLE_USER_METRICS

      @Private public static final String ENABLE_USER_METRICS
      See Also:

    • DEFAULT_ENABLE_USER_METRICS

      @Private public static final boolean DEFAULT_ENABLE_USER_METRICS
      See Also:

    • RESOURCE_CALCULATOR_CLASS

      @Private public static final String RESOURCE_CALCULATOR_CLASS
      ResourceComparator for scheduling.
      See Also:

    • DEFAULT_RESOURCE_CALCULATOR_CLASS

      @Private public static final Class<? extends org.apache.hadoop.yarn.util.resource.ResourceCalculator> DEFAULT_RESOURCE_CALCULATOR_CLASS

    • ROOT

      @Private public static final String ROOT
      See Also:

    • NODE_LOCALITY_DELAY

      @Private public static final String NODE_LOCALITY_DELAY
      See Also:

    • DEFAULT_NODE_LOCALITY_DELAY

      @Private public static final int DEFAULT_NODE_LOCALITY_DELAY
      See Also:

    • RACK_LOCALITY_ADDITIONAL_DELAY

      @Private public static final String RACK_LOCALITY_ADDITIONAL_DELAY
      See Also:

    • DEFAULT_RACK_LOCALITY_ADDITIONAL_DELAY

      @Private public static final int DEFAULT_RACK_LOCALITY_ADDITIONAL_DELAY
      See Also:

    • RACK_LOCALITY_FULL_RESET

      @Private public static final String RACK_LOCALITY_FULL_RESET
      See Also:

    • DEFAULT_OFFSWITCH_PER_HEARTBEAT_LIMIT

      @Private public static final int DEFAULT_OFFSWITCH_PER_HEARTBEAT_LIMIT
      See Also:

    • OFFSWITCH_PER_HEARTBEAT_LIMIT

      @Private public static final String OFFSWITCH_PER_HEARTBEAT_LIMIT
      See Also:

    • DEFAULT_RACK_LOCALITY_FULL_RESET

      @Private public static final boolean DEFAULT_RACK_LOCALITY_FULL_RESET
      See Also:

    • SCHEDULE_ASYNCHRONOUSLY_PREFIX

      @Private public static final String SCHEDULE_ASYNCHRONOUSLY_PREFIX
      See Also:

    • SCHEDULE_ASYNCHRONOUSLY_ENABLE

      @Private public static final String SCHEDULE_ASYNCHRONOUSLY_ENABLE
      See Also:

    • SCHEDULE_ASYNCHRONOUSLY_MAXIMUM_THREAD

      @Private public static final String SCHEDULE_ASYNCHRONOUSLY_MAXIMUM_THREAD
      See Also:

    • SCHEDULE_ASYNCHRONOUSLY_MAXIMUM_PENDING_BACKLOGS

      @Private public static final String SCHEDULE_ASYNCHRONOUSLY_MAXIMUM_PENDING_BACKLOGS
      See Also:

    • SCHEDULE_ASYNCHRONOUSLY_INTERVAL

      @Private public static final String SCHEDULE_ASYNCHRONOUSLY_INTERVAL
      See Also:

    • DEFAULT_SCHEDULE_ASYNCHRONOUSLY_INTERVAL

      @Private public static final long DEFAULT_SCHEDULE_ASYNCHRONOUSLY_INTERVAL
      See Also:

    • APP_FAIL_FAST

      @Private public static final String APP_FAIL_FAST
      See Also:

    • DEFAULT_APP_FAIL_FAST

      @Private public static final boolean DEFAULT_APP_FAIL_FAST
      See Also:

    • DEFAULT_SCHEDULE_ASYNCHRONOUSLY_MAXIMUM_PENDING_BACKLOGS

      @Private public static final Integer DEFAULT_SCHEDULE_ASYNCHRONOUSLY_MAXIMUM_PENDING_BACKLOGS

    • DEFAULT_SCHEDULE_ASYNCHRONOUSLY_ENABLE

      @Private public static final boolean DEFAULT_SCHEDULE_ASYNCHRONOUSLY_ENABLE
      See Also:

    • QUEUE_MAPPING

      @Private public static final String QUEUE_MAPPING
      See Also:

    • QUEUE_MAPPING_NAME

      @Private public static final String QUEUE_MAPPING_NAME
      See Also:

    • ENABLE_QUEUE_MAPPING_OVERRIDE

      @Private public static final String ENABLE_QUEUE_MAPPING_OVERRIDE
      See Also:

    • DEFAULT_ENABLE_QUEUE_MAPPING_OVERRIDE

      @Private public static final boolean DEFAULT_ENABLE_QUEUE_MAPPING_OVERRIDE
      See Also:

    • WORKFLOW_PRIORITY_MAPPINGS

      @Private public static final String WORKFLOW_PRIORITY_MAPPINGS
      See Also:

    • ENABLE_WORKFLOW_PRIORITY_MAPPINGS_OVERRIDE

      @Private public static final String ENABLE_WORKFLOW_PRIORITY_MAPPINGS_OVERRIDE
      See Also:

    • DEFAULT_ENABLE_WORKFLOW_PRIORITY_MAPPINGS_OVERRIDE

      @Private public static final boolean DEFAULT_ENABLE_WORKFLOW_PRIORITY_MAPPINGS_OVERRIDE
      See Also:

    • QUEUE_PREEMPTION_DISABLED

      @Private public static final String QUEUE_PREEMPTION_DISABLED
      See Also:

    • DEFAULT_APPLICATION_PRIORITY

      @Private public static final String DEFAULT_APPLICATION_PRIORITY
      See Also:

    • DEFAULT_CONFIGURATION_APPLICATION_PRIORITY

      @Private public static final Integer DEFAULT_CONFIGURATION_APPLICATION_PRIORITY

    • AVERAGE_CAPACITY

      @Private public static final String AVERAGE_CAPACITY
      See Also:

    • IS_RESERVABLE

      @Private public static final String IS_RESERVABLE
      See Also:

    • RESERVATION_WINDOW

      @Private public static final String RESERVATION_WINDOW
      See Also:

    • INSTANTANEOUS_MAX_CAPACITY

      @Private public static final String INSTANTANEOUS_MAX_CAPACITY
      See Also:

    • RESERVATION_ADMISSION_POLICY

      @Private public static final String RESERVATION_ADMISSION_POLICY
      See Also:

    • RESERVATION_AGENT_NAME

      @Private public static final String RESERVATION_AGENT_NAME
      See Also:

    • RESERVATION_SHOW_RESERVATION_AS_QUEUE

      @Private public static final String RESERVATION_SHOW_RESERVATION_AS_QUEUE
      See Also:

    • RESERVATION_PLANNER_NAME

      @Private public static final String RESERVATION_PLANNER_NAME
      See Also:

    • RESERVATION_MOVE_ON_EXPIRY

      @Private public static final String RESERVATION_MOVE_ON_EXPIRY
      See Also:

    • RESERVATION_ENFORCEMENT_WINDOW

      @Private public static final String RESERVATION_ENFORCEMENT_WINDOW
      See Also:

    • LAZY_PREEMPTION_ENABLED

      @Private public static final String LAZY_PREEMPTION_ENABLED
      See Also:

    • DEFAULT_LAZY_PREEMPTION_ENABLED

      @Private public static final boolean DEFAULT_LAZY_PREEMPTION_ENABLED
      See Also:

    • ASSIGN_MULTIPLE_ENABLED

      @Private public static final String ASSIGN_MULTIPLE_ENABLED
      See Also:

    • DEFAULT_ASSIGN_MULTIPLE_ENABLED

      @Private public static final boolean DEFAULT_ASSIGN_MULTIPLE_ENABLED
      See Also:

    • MAX_ASSIGN_PER_HEARTBEAT

      @Private public static final String MAX_ASSIGN_PER_HEARTBEAT
      Maximum number of containers to assign on each check-in.
      See Also:

    • DEFAULT_MAX_ASSIGN_PER_HEARTBEAT

      @Private public static final int DEFAULT_MAX_ASSIGN_PER_HEARTBEAT
      Avoid potential risk that greedy assign multiple may involve
      See Also:

    • MINIMUM_RESOURCE

      @Private public static final String MINIMUM_RESOURCE
      Configuring absolute min/max resources in a queue.
      See Also:

    • MAXIMUM_RESOURCE

      @Private public static final String MAXIMUM_RESOURCE
      See Also:

    • DEFAULT_RESOURCE_TYPES

      public static final String DEFAULT_RESOURCE_TYPES
      See Also:

    • PATTERN_FOR_ABSOLUTE_RESOURCE

      public static final String PATTERN_FOR_ABSOLUTE_RESOURCE
      See Also:

    • RESOURCE_PATTERN

      public static final Pattern RESOURCE_PATTERN

    • MAX_PARALLEL_APPLICATIONS

      public static final String MAX_PARALLEL_APPLICATIONS
      See Also:

    • DEFAULT_MAX_PARALLEL_APPLICATIONS

      public static final int DEFAULT_MAX_PARALLEL_APPLICATIONS
      See Also:

    • ALLOW_ZERO_CAPACITY_SUM

      public static final String ALLOW_ZERO_CAPACITY_SUM
      See Also:

    • DEFAULT_ALLOW_ZERO_CAPACITY_SUM

      public static final boolean DEFAULT_ALLOW_ZERO_CAPACITY_SUM
      See Also:

    • MAPPING_RULE_FORMAT

      public static final String MAPPING_RULE_FORMAT
      See Also:

    • MAPPING_RULE_JSON

      public static final String MAPPING_RULE_JSON
      See Also:

    • MAPPING_RULE_JSON_FILE

      public static final String MAPPING_RULE_JSON_FILE
      See Also:

    • MAPPING_RULE_FORMAT_LEGACY

      public static final String MAPPING_RULE_FORMAT_LEGACY
      See Also:

    • MAPPING_RULE_FORMAT_JSON

      public static final String MAPPING_RULE_FORMAT_JSON
      See Also:

    • MAPPING_RULE_FORMAT_DEFAULT

      public static final String MAPPING_RULE_FORMAT_DEFAULT
      See Also:

    • DEFAULT_LEGACY_QUEUE_MODE

      public static final boolean DEFAULT_LEGACY_QUEUE_MODE
      See Also:

    • PREEMPTION_OBSERVE_ONLY

      public static final String PREEMPTION_OBSERVE_ONLY
      If true, run the policy but do not affect the cluster with preemption and kill events.
      See Also:

    • DEFAULT_PREEMPTION_OBSERVE_ONLY

      public static final boolean DEFAULT_PREEMPTION_OBSERVE_ONLY
      See Also:

    • PREEMPTION_MONITORING_INTERVAL

      public static final String PREEMPTION_MONITORING_INTERVAL
      Time in milliseconds between invocations of this policy
      See Also:

    • DEFAULT_PREEMPTION_MONITORING_INTERVAL

      public static final long DEFAULT_PREEMPTION_MONITORING_INTERVAL
      See Also:

    • PREEMPTION_WAIT_TIME_BEFORE_KILL

      public static final String PREEMPTION_WAIT_TIME_BEFORE_KILL
      Time in milliseconds between requesting a preemption from an application and killing the container.
      See Also:

    • DEFAULT_PREEMPTION_WAIT_TIME_BEFORE_KILL

      public static final long DEFAULT_PREEMPTION_WAIT_TIME_BEFORE_KILL
      See Also:

    • TOTAL_PREEMPTION_PER_ROUND

      public static final String TOTAL_PREEMPTION_PER_ROUND
      Maximum percentage of resources preemptionCandidates in a single round. By controlling this value one can throttle the pace at which containers are reclaimed from the cluster. After computing the total desired preemption, the policy scales it back within this limit.
      See Also:

    • DEFAULT_TOTAL_PREEMPTION_PER_ROUND

      public static final float DEFAULT_TOTAL_PREEMPTION_PER_ROUND
      See Also:

    • PREEMPTION_MAX_IGNORED_OVER_CAPACITY

      public static final String PREEMPTION_MAX_IGNORED_OVER_CAPACITY
      Maximum amount of resources above the target capacity ignored for preemption. This defines a deadzone around the target capacity that helps prevent thrashing and oscillations around the computed target balance. High values would slow the time to capacity and (absent natural completions) it might prevent convergence to guaranteed capacity.
      See Also:

    • DEFAULT_PREEMPTION_MAX_IGNORED_OVER_CAPACITY

      public static final double DEFAULT_PREEMPTION_MAX_IGNORED_OVER_CAPACITY
      See Also:

    • PREEMPTION_NATURAL_TERMINATION_FACTOR

      public static final String PREEMPTION_NATURAL_TERMINATION_FACTOR
      Given a computed preemption target, account for containers naturally expiring and preempt only this percentage of the delta. This determines the rate of geometric convergence into the deadzone (PREEMPTION_MAX_IGNORED_OVER_CAPACITY). For example, a termination factor of 0.5 will reclaim almost 95% of resources within 5 * PREEMPTION_WAIT_TIME_BEFORE_KILL, even absent natural termination.
      See Also:

    • DEFAULT_PREEMPTION_NATURAL_TERMINATION_FACTOR

      public static final double DEFAULT_PREEMPTION_NATURAL_TERMINATION_FACTOR
      See Also:

    • ADDITIONAL_RESOURCE_BALANCE_BASED_ON_RESERVED_CONTAINERS

      public static final String ADDITIONAL_RESOURCE_BALANCE_BASED_ON_RESERVED_CONTAINERS
      By default, reserved resource will be excluded while balancing capacities of queues. Why doing this? In YARN-4390, we added preemption-based-on-reserved-container Support. To reduce unnecessary preemption for large containers. We will not include reserved resources while calculating ideal-allocation in FifoCandidatesSelector. Changes in YARN-4390 will significantly reduce number of containers preempted When cluster has heterogeneous container requests. (Please check test report: https://issues.apache.org/jira/secure/attachment/12796197/YARN-4390-test-results.pdf However, on the other hand, in some corner cases, especially for fragmented cluster. It could lead to preemption cannot kick in in some cases. Please see YARN-5731. So to solve the problem, make this change to be configurable, and please note that it is an experimental option.
      See Also:

    • DEFAULT_ADDITIONAL_RESOURCE_BALANCE_BASED_ON_RESERVED_CONTAINERS

      public static final boolean DEFAULT_ADDITIONAL_RESOURCE_BALANCE_BASED_ON_RESERVED_CONTAINERS
      See Also:

    • PREEMPTION_SELECT_CANDIDATES_FOR_RESERVED_CONTAINERS

      public static final String PREEMPTION_SELECT_CANDIDATES_FOR_RESERVED_CONTAINERS
      When calculating which containers to be preempted, we will try to preempt containers for reserved containers first. By default is false.
      See Also:

    • DEFAULT_PREEMPTION_SELECT_CANDIDATES_FOR_RESERVED_CONTAINERS

      public static final boolean DEFAULT_PREEMPTION_SELECT_CANDIDATES_FOR_RESERVED_CONTAINERS
      See Also:

    • INTRAQUEUE_PREEMPTION_ENABLED

      public static final String INTRAQUEUE_PREEMPTION_ENABLED
      For intra-queue preemption, priority/user-limit/fairness based selectors can help to preempt containers.
      See Also:

    • DEFAULT_INTRAQUEUE_PREEMPTION_ENABLED

      public static final boolean DEFAULT_INTRAQUEUE_PREEMPTION_ENABLED
      See Also:

    • INTRAQUEUE_PREEMPTION_MINIMUM_THRESHOLD

      public static final String INTRAQUEUE_PREEMPTION_MINIMUM_THRESHOLD
      For intra-queue preemption, consider those queues which are above used cap limit.
      See Also:

    • DEFAULT_INTRAQUEUE_PREEMPTION_MINIMUM_THRESHOLD

      public static final float DEFAULT_INTRAQUEUE_PREEMPTION_MINIMUM_THRESHOLD
      See Also:

    • INTRAQUEUE_PREEMPTION_MAX_ALLOWABLE_LIMIT

      public static final String INTRAQUEUE_PREEMPTION_MAX_ALLOWABLE_LIMIT
      For intra-queue preemption, allowable maximum-preemptable limit per queue.
      See Also:

    • DEFAULT_INTRAQUEUE_PREEMPTION_MAX_ALLOWABLE_LIMIT

      public static final float DEFAULT_INTRAQUEUE_PREEMPTION_MAX_ALLOWABLE_LIMIT
      See Also:

    • INTRAQUEUE_PREEMPTION_ORDER_POLICY

      public static final String INTRAQUEUE_PREEMPTION_ORDER_POLICY
      For intra-queue preemption, enforce a preemption order such as "userlimit_first" or "priority_first".
      See Also:

    • DEFAULT_INTRAQUEUE_PREEMPTION_ORDER_POLICY

      public static final String DEFAULT_INTRAQUEUE_PREEMPTION_ORDER_POLICY
      See Also:

    • CROSS_QUEUE_PREEMPTION_CONSERVATIVE_DRF

      public static final String CROSS_QUEUE_PREEMPTION_CONSERVATIVE_DRF
      Flag to determine whether or not to preempt containers from apps where some used resources are less than the user's user limit.
      See Also:

    • DEFAULT_CROSS_QUEUE_PREEMPTION_CONSERVATIVE_DRF

      public static final Boolean DEFAULT_CROSS_QUEUE_PREEMPTION_CONSERVATIVE_DRF

    • IN_QUEUE_PREEMPTION_CONSERVATIVE_DRF

      public static final String IN_QUEUE_PREEMPTION_CONSERVATIVE_DRF
      See Also:

    • DEFAULT_IN_QUEUE_PREEMPTION_CONSERVATIVE_DRF

      public static final Boolean DEFAULT_IN_QUEUE_PREEMPTION_CONSERVATIVE_DRF

    • PREEMPTION_TO_BALANCE_QUEUES_BEYOND_GUARANTEED

      public static final String PREEMPTION_TO_BALANCE_QUEUES_BEYOND_GUARANTEED
      Should we allow queues continue grow after all queue reaches their guaranteed capacity.
      See Also:

    • DEFAULT_PREEMPTION_TO_BALANCE_QUEUES_BEYOND_GUARANTEED

      public static final boolean DEFAULT_PREEMPTION_TO_BALANCE_QUEUES_BEYOND_GUARANTEED
      See Also:

    • MAX_WAIT_BEFORE_KILL_FOR_QUEUE_BALANCE_PREEMPTION

      public static final String MAX_WAIT_BEFORE_KILL_FOR_QUEUE_BALANCE_PREEMPTION
      How long we will wait to balance queues, by default it is 5 mins.
      See Also:

    • DEFAULT_MAX_WAIT_BEFORE_KILL_FOR_QUEUE_BALANCE_PREEMPTION

      public static final long DEFAULT_MAX_WAIT_BEFORE_KILL_FOR_QUEUE_BALANCE_PREEMPTION
      See Also:

    • QUEUE_GLOBAL_MAX_APPLICATION

      @Private public static final String QUEUE_GLOBAL_MAX_APPLICATION
      Maximum application for a queue to be used when application per queue is not defined.To be consistent with previous version the default value is set as UNDEFINED.
      See Also:

    • QUEUE_UTILIZATION_ORDERING_POLICY

      public static final String QUEUE_UTILIZATION_ORDERING_POLICY
      Less relative usage queue can get next resource, this is default
      See Also:

    • QUEUE_PRIORITY_UTILIZATION_ORDERING_POLICY

      public static final String QUEUE_PRIORITY_UTILIZATION_ORDERING_POLICY
      Combination of relative usage and priority
      See Also:

    • DEFAULT_QUEUE_ORDERING_POLICY

      public static final String DEFAULT_QUEUE_ORDERING_POLICY
      See Also:

    • MAXIMUM_LIFETIME_SUFFIX

      public static final String MAXIMUM_LIFETIME_SUFFIX
      See Also:

    • DEFAULT_LIFETIME_SUFFIX

      public static final String DEFAULT_LIFETIME_SUFFIX
      See Also:

    • DEFAULT_AUTO_CREATE_CHILD_QUEUE_ENABLED

      @Private public static final boolean DEFAULT_AUTO_CREATE_CHILD_QUEUE_ENABLED
      See Also:

    • AUTO_CREATE_CHILD_QUEUE_ENABLED

      @Private public static final String AUTO_CREATE_CHILD_QUEUE_ENABLED
      See Also:

    • AUTO_QUEUE_CREATION_V2_PREFIX

      @Private protected static final String AUTO_QUEUE_CREATION_V2_PREFIX
      See Also:

    • AUTO_QUEUE_CREATION_V2_ENABLED

      @Private public static final String AUTO_QUEUE_CREATION_V2_ENABLED
      See Also:

    • AUTO_QUEUE_CREATION_V2_MAX_QUEUES

      @Private public static final String AUTO_QUEUE_CREATION_V2_MAX_QUEUES
      See Also:

    • DEFAULT_AUTO_QUEUE_CREATION_V2_MAX_QUEUES

      @Private public static final int DEFAULT_AUTO_QUEUE_CREATION_V2_MAX_QUEUES
      See Also:

    • MAXIMUM_QUEUE_DEPTH

      @Private public static final String MAXIMUM_QUEUE_DEPTH
      See Also:

    • DEFAULT_MAXIMUM_QUEUE_DEPTH

      @Private public static final int DEFAULT_MAXIMUM_QUEUE_DEPTH
      See Also:

    • DEFAULT_AUTO_QUEUE_CREATION_ENABLED

      @Private public static final boolean DEFAULT_AUTO_QUEUE_CREATION_ENABLED
      See Also:

    • AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX

      @Private public static final String AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX
      See Also:

    • AUTO_CREATE_QUEUE_MAX_QUEUES

      @Private public static final String AUTO_CREATE_QUEUE_MAX_QUEUES
      See Also:

    • DEFAULT_AUTO_CREATE_QUEUE_MAX_QUEUES

      @Private public static final int DEFAULT_AUTO_CREATE_QUEUE_MAX_QUEUES
      See Also:

    • FAIL_AUTO_CREATION_ON_EXCEEDING_CAPACITY

      @Private public static final String FAIL_AUTO_CREATION_ON_EXCEEDING_CAPACITY
      See Also:

    • DEFAULT_FAIL_AUTO_CREATION_ON_EXCEEDING_CAPACITY

      @Private public static final boolean DEFAULT_FAIL_AUTO_CREATION_ON_EXCEEDING_CAPACITY
      See Also:

    • AUTO_CREATED_QUEUE_MANAGEMENT_POLICY

      @Private public static final String AUTO_CREATED_QUEUE_MANAGEMENT_POLICY
      See Also:

    • DEFAULT_AUTO_CREATED_QUEUE_MANAGEMENT_POLICY

      @Private public static final String DEFAULT_AUTO_CREATED_QUEUE_MANAGEMENT_POLICY
      See Also:

    • QUEUE_MANAGEMENT_MONITORING_INTERVAL

      @Private public static final String QUEUE_MANAGEMENT_MONITORING_INTERVAL
      Time in milliseconds between invocations of this policy
      See Also:

    • DEFAULT_QUEUE_MANAGEMENT_MONITORING_INTERVAL

      @Private public static final long DEFAULT_QUEUE_MANAGEMENT_MONITORING_INTERVAL
      See Also:

    • DEFAULT_AUTO_CREATE_CHILD_QUEUE_AUTO_REMOVAL_ENABLE

      @Private public static final boolean DEFAULT_AUTO_CREATE_CHILD_QUEUE_AUTO_REMOVAL_ENABLE
      See Also:

    • AUTO_CREATE_CHILD_QUEUE_AUTO_REMOVAL_ENABLE

      @Private public static final String AUTO_CREATE_CHILD_QUEUE_AUTO_REMOVAL_ENABLE
      See Also:

    • DEFAULT_AUTO_CREATE_CHILD_QUEUE_EXPIRED_TIME

      @Private public static final long DEFAULT_AUTO_CREATE_CHILD_QUEUE_EXPIRED_TIME
      See Also:

    • AUTO_CREATE_CHILD_QUEUE_EXPIRED_TIME

      @Private public static final String AUTO_CREATE_CHILD_QUEUE_EXPIRED_TIME
      See Also:

    • QUEUE_AUTO_REFRESH_MONITORING_INTERVAL

      @Private public static final String QUEUE_AUTO_REFRESH_MONITORING_INTERVAL
      Time in milliseconds between invocations of QueueConfigurationAutoRefreshPolicy.
      See Also:

    • DEFAULT_QUEUE_AUTO_REFRESH_MONITORING_INTERVAL

      @Private public static final long DEFAULT_QUEUE_AUTO_REFRESH_MONITORING_INTERVAL
      See Also:

    • MULTI_NODE_SORTING_POLICIES

      @Private public static final String MULTI_NODE_SORTING_POLICIES
      See Also:

    • MULTI_NODE_SORTING_POLICY_NAME

      @Private public static final String MULTI_NODE_SORTING_POLICY_NAME
      See Also:

    • DEFAULT_NODE_SORTING_POLICY

      public static final String DEFAULT_NODE_SORTING_POLICY
      resource usage based node sorting algorithm.
      See Also:

    • DEFAULT_NODE_SORTING_POLICY_CLASSNAME

      public static final String DEFAULT_NODE_SORTING_POLICY_CLASSNAME
      See Also:

    • DEFAULT_MULTI_NODE_SORTING_INTERVAL

      public static final long DEFAULT_MULTI_NODE_SORTING_INTERVAL
      See Also:

    • MULTI_NODE_PLACEMENT_ENABLED

      @Private public static final String MULTI_NODE_PLACEMENT_ENABLED
      See Also:

    • DEFAULT_MULTI_NODE_PLACEMENT_ENABLED

      @Private public static final boolean DEFAULT_MULTI_NODE_PLACEMENT_ENABLED
      See Also:

  • Constructor Details

    • CapacitySchedulerConfiguration

      public CapacitySchedulerConfiguration()

    • CapacitySchedulerConfiguration

      public CapacitySchedulerConfiguration(org.apache.hadoop.conf.Configuration configuration)

    • CapacitySchedulerConfiguration

      public CapacitySchedulerConfiguration(org.apache.hadoop.conf.Configuration configuration, boolean useLocalConfigurationProvider)

  • Method Details

    • getQueueCapacityConfigParser

      public static QueueCapacityConfigParser getQueueCapacityConfigParser()

    • getMaximumAutoCreatedQueueDepth

      public int getMaximumAutoCreatedQueueDepth(QueuePath queuePath)

    • setMaximumAutoCreatedQueueDepth

      public void setMaximumAutoCreatedQueueDepth(QueuePath queue, int value)

    • setMaximumAutoCreatedQueueDepth

      public void setMaximumAutoCreatedQueueDepth(int value)

    • setMaximumSystemApplications

      public void setMaximumSystemApplications(int numMaxApps)

    • getMaximumSystemApplications

      public int getMaximumSystemApplications()

    • setMaximumApplicationMasterResourcePercent

      public void setMaximumApplicationMasterResourcePercent(float percent)

    • getMaximumApplicationMasterResourcePercent

      public float getMaximumApplicationMasterResourcePercent()

    • getMaximumApplicationsPerQueue

      public int getMaximumApplicationsPerQueue(QueuePath queue)
      Get the maximum applications per queue setting.
      Parameters:
      queue - path of the queue
      Returns:
      setting specified or -1 if not set

    • setMaximumApplicationsPerQueue

      @VisibleForTesting public void setMaximumApplicationsPerQueue(QueuePath queue, int numMaxApps)

    • getMaximumApplicationMasterResourcePerQueuePercent

      public float getMaximumApplicationMasterResourcePerQueuePercent(QueuePath queue)
      Get the maximum am resource percent per queue setting.
      Parameters:
      queue - path of the queue
      Returns:
      per queue setting or defaults to the global am-resource-percent setting if per queue setting not present

    • setMaximumApplicationMasterResourcePerQueuePercent

      public void setMaximumApplicationMasterResourcePerQueuePercent(QueuePath queue, float percent)

    • getNonLabeledQueueWeight

      public float getNonLabeledQueueWeight(QueuePath queue)

    • setNonLabeledQueueWeight

      public void setNonLabeledQueueWeight(QueuePath queue, float weight)

    • setLabeledQueueWeight

      public void setLabeledQueueWeight(QueuePath queue, String label, float weight)

    • getLabeledQueueWeight

      public float getLabeledQueueWeight(QueuePath queue, String label)

    • getNonLabeledQueueCapacity

      public float getNonLabeledQueueCapacity(QueuePath queue)

    • setCapacity

      public void setCapacity(QueuePath queue, float capacity)

    • setCapacity

      @VisibleForTesting public void setCapacity(QueuePath queue, String absoluteResourceCapacity)

    • getNonLabeledQueueMaximumCapacity

      public float getNonLabeledQueueMaximumCapacity(QueuePath queue)

    • setMaximumCapacity

      public void setMaximumCapacity(QueuePath queue, float maxCapacity)

    • setCapacityByLabel

      public void setCapacityByLabel(QueuePath queue, String label, float capacity)

    • setCapacityByLabel

      @VisibleForTesting public void setCapacityByLabel(QueuePath queue, String label, String absoluteResourceCapacity)

    • setMaximumCapacityByLabel

      public void setMaximumCapacityByLabel(QueuePath queue, String label, float capacity)

    • setMaximumCapacityByLabel

      public void setMaximumCapacityByLabel(QueuePath queue, String label, String absoluteResourceCapacity)

    • getUserLimit

      public float getUserLimit(QueuePath queue)

    • getAppOrderingPolicy

      public <S extends SchedulableEntity> OrderingPolicy<S> getAppOrderingPolicy(QueuePath queue)

    • setUserLimit

      public void setUserLimit(QueuePath queue, float userLimit)

    • setDefaultUserLimit

      @VisibleForTesting public void setDefaultUserLimit(float defaultUserLimit)

    • getUserLimitFactor

      public float getUserLimitFactor(QueuePath queue)

    • setUserLimitFactor

      public void setUserLimitFactor(QueuePath queuePath, float userLimitFactor)

    • setDefaultUserLimitFactor

      @VisibleForTesting public void setDefaultUserLimitFactor(float defaultUserLimitFactor)

    • getConfiguredState

      public org.apache.hadoop.yarn.api.records.QueueState getConfiguredState(QueuePath queue)

    • getState

      public org.apache.hadoop.yarn.api.records.QueueState getState(QueuePath queue)

    • setState

      @Private @VisibleForTesting public void setState(QueuePath queue, org.apache.hadoop.yarn.api.records.QueueState state)

    • setAccessibleNodeLabels

      public void setAccessibleNodeLabels(QueuePath queue, Set<String> labels)

    • getAccessibleNodeLabels

      public Set<String> getAccessibleNodeLabels(QueuePath queue)

    • setCapacityVector

      public void setCapacityVector(QueuePath queuePath, String label, String capacityVector)

    • setMaximumCapacityVector

      public void setMaximumCapacityVector(QueuePath queuePath, String label, String capacityVector)

    • getLabeledQueueCapacity

      public float getLabeledQueueCapacity(QueuePath queue, String label)

    • getLabeledQueueMaximumCapacity

      public float getLabeledQueueMaximumCapacity(QueuePath queue, String label)

    • getDefaultNodeLabelExpression

      public String getDefaultNodeLabelExpression(QueuePath queue)

    • setDefaultNodeLabelExpression

      public void setDefaultNodeLabelExpression(QueuePath queue, String exp)

    • getMaximumAMResourcePercentPerPartition

      public float getMaximumAMResourcePercentPerPartition(QueuePath queue, String label)

    • setMaximumAMResourcePercentPerPartition

      public void setMaximumAMResourcePercentPerPartition(QueuePath queue, String label, float percent)

    • getReservationContinueLook

      public boolean getReservationContinueLook()

    • getSkipAllocateOnNodesWithReservedContainer

      public boolean getSkipAllocateOnNodesWithReservedContainer()

    • getAcl

      public org.apache.hadoop.security.authorize.AccessControlList getAcl(QueuePath queue, org.apache.hadoop.yarn.api.records.QueueACL acl)

    • setAcl

      public void setAcl(QueuePath queue, org.apache.hadoop.yarn.api.records.QueueACL acl, String aclString)

    • getACLsForLegacyAutoCreatedLeafQueue

      public Map<org.apache.hadoop.yarn.security.AccessType,org.apache.hadoop.security.authorize.AccessControlList> getACLsForLegacyAutoCreatedLeafQueue(QueuePath parentQueuePath)
      Creates a mapping of queue ACLs for a Legacy Auto Created Leaf Queue.
      Parameters:
      parentQueuePath - the parent's queue path
      Returns:
      A mapping of the queue ACLs.

    • getACLsForFlexibleAutoCreatedParentQueue

      public static Map<org.apache.hadoop.yarn.security.AccessType,org.apache.hadoop.security.authorize.AccessControlList> getACLsForFlexibleAutoCreatedParentQueue(AutoCreatedQueueTemplate aqc)
      Creates a mapping of queue ACLs for a Flexible Auto Created Parent Queue. The .parent-template is preferred to .template ACLs.
      Parameters:
      aqc - The AQC templates to use.
      Returns:
      A mapping of the queue ACLs.

    • getACLsForFlexibleAutoCreatedLeafQueue

      public static Map<org.apache.hadoop.yarn.security.AccessType,org.apache.hadoop.security.authorize.AccessControlList> getACLsForFlexibleAutoCreatedLeafQueue(AutoCreatedQueueTemplate aqc)
      Creates a mapping of queue ACLs for a Flexible Auto Created Leaf Queue. The .leaf-template is preferred to .template ACLs.
      Parameters:
      aqc - The AQC templates to use.
      Returns:
      A mapping of the queue ACLs.

    • getReservationAcls

      public Map<org.apache.hadoop.yarn.api.records.ReservationACL,org.apache.hadoop.security.authorize.AccessControlList> getReservationAcls(QueuePath queue)
      Description copied from class: ReservationSchedulerConfiguration
      Gets a map containing the AccessControlList of users for each ReservationACL acl on thee specified queue.
      Specified by:
      getReservationAcls in class ReservationSchedulerConfiguration
      Parameters:
      queue - the queue with which to check a user's permissions.
      Returns:
      The a Map of ReservationACL to AccessControlList which contains a list of users that have the specified permission level.

    • getAcls

      public Map<org.apache.hadoop.yarn.security.AccessType,org.apache.hadoop.security.authorize.AccessControlList> getAcls(QueuePath queue)

    • setAcls

      public void setAcls(QueuePath queue, Map<org.apache.hadoop.yarn.api.records.QueueACL,org.apache.hadoop.security.authorize.AccessControlList> acls)

    • setReservationAcls

      @VisibleForTesting public void setReservationAcls(QueuePath queue, Map<org.apache.hadoop.yarn.api.records.ReservationACL,org.apache.hadoop.security.authorize.AccessControlList> acls)

    • setPriorityAcls

      @VisibleForTesting public void setPriorityAcls(QueuePath queue, org.apache.hadoop.yarn.api.records.Priority priority, org.apache.hadoop.yarn.api.records.Priority defaultPriority, String[] acls)

    • getPriorityAcls

      public List<AppPriorityACLGroup> getPriorityAcls(QueuePath queue, org.apache.hadoop.yarn.api.records.Priority clusterMaxPriority)

    • getQueues

      public List<String> getQueues(QueuePath queue)

    • setQueues

      public void setQueues(QueuePath queue, String[] subQueues)

    • getMinimumAllocation

      public org.apache.hadoop.yarn.api.records.Resource getMinimumAllocation()

    • getQueuePriority

      @Private public org.apache.hadoop.yarn.api.records.Priority getQueuePriority(QueuePath queue)

    • setQueuePriority

      @Private public void setQueuePriority(QueuePath queue, int priority)

    • getQueueMaximumAllocation

      public org.apache.hadoop.yarn.api.records.Resource getQueueMaximumAllocation(QueuePath queue)
      Get maximum_allocation setting for the specified queue from the configuration.
      Parameters:
      queue - name of the queue
      Returns:
      Resource object or Resource.none if not set

    • setQueueMaximumAllocation

      public void setQueueMaximumAllocation(QueuePath queue, String maximumAllocation)

    • getConfigurationProperties

      public ConfigurationProperties getConfigurationProperties()
      Get all configuration properties parsed in a ConfigurationProperties object.
      Returns:
      configuration properties

    • reinitializeConfigurationProperties

      public void reinitializeConfigurationProperties()
      Reinitializes the cached ConfigurationProperties object.

    • setQueueMaximumAllocationMb

      public void setQueueMaximumAllocationMb(QueuePath queue, int value)

    • setQueueMaximumAllocationVcores

      public void setQueueMaximumAllocationVcores(QueuePath queue, int value)

    • getQueueMaximumAllocationMb

      public long getQueueMaximumAllocationMb(QueuePath queue)

    • getQueueMaximumAllocationVcores

      public int getQueueMaximumAllocationVcores(QueuePath queue)

    • getEnableUserMetrics

      public boolean getEnableUserMetrics()

    • getOffSwitchPerHeartbeatLimit

      public int getOffSwitchPerHeartbeatLimit()

    • setOffSwitchPerHeartbeatLimit

      public void setOffSwitchPerHeartbeatLimit(int limit)

    • getNodeLocalityDelay

      public int getNodeLocalityDelay()

    • setNodeLocalityDelay

      @VisibleForTesting public void setNodeLocalityDelay(int nodeLocalityDelay)

    • getRackLocalityAdditionalDelay

      public int getRackLocalityAdditionalDelay()

    • getRackLocalityFullReset

      public boolean getRackLocalityFullReset()

    • getResourceCalculator

      public org.apache.hadoop.yarn.util.resource.ResourceCalculator getResourceCalculator()

    • getUsePortForNodeName

      public boolean getUsePortForNodeName()

    • setResourceComparator

      public void setResourceComparator(Class<? extends org.apache.hadoop.yarn.util.resource.ResourceCalculator> resourceCalculatorClass)

    • getScheduleAynschronously

      public boolean getScheduleAynschronously()

    • setScheduleAynschronously

      public void setScheduleAynschronously(boolean async)

    • getOverrideWithQueueMappings

      public boolean getOverrideWithQueueMappings()

    • setOverrideWithQueueMappings

      @Private @VisibleForTesting public void setOverrideWithQueueMappings(boolean overrideWithQueueMappings)

    • getQueueMappingEntity

      public List<QueueMapping> getQueueMappingEntity(String queueMappingSuffix)

    • setQueueMappingEntities

      @VisibleForTesting public void setQueueMappingEntities(List<QueueMapping> queueMappings, String queueMappingSuffix)

    • getOverrideWithWorkflowPriorityMappings

      public boolean getOverrideWithWorkflowPriorityMappings()

    • getWorkflowPriorityMappings

      public Collection<String> getWorkflowPriorityMappings()

    • getQueueMappings

      public List<QueueMapping> getQueueMappings()
      Get user/group mappings to queues.
      Returns:
      user/groups mappings or null on illegal configs

    • parseLegacyMappingRules

      public List<MappingRule> parseLegacyMappingRules()

    • parseJSONMappingRules

      public List<MappingRule> parseJSONMappingRules() throws IOException
      Throws:
      IOException

    • setMappingRuleFormat

      public void setMappingRuleFormat(String format)

    • setMappingRuleJson

      public void setMappingRuleJson(String json)

    • getMappingRules

      public List<MappingRule> getMappingRules() throws IOException
      Throws:
      IOException

    • setQueuePlacementRules

      @Private @VisibleForTesting public void setQueuePlacementRules(Collection<String> queuePlacementRules)

    • setQueueMappings

      @Private @VisibleForTesting public void setQueueMappings(List<QueueMapping> queueMappings)

    • setAppNameMappings

      @Private @VisibleForTesting public void setAppNameMappings(List<QueueMapping> queueMappings)

    • isReservable

      public boolean isReservable(QueuePath queue)
      Description copied from class: ReservationSchedulerConfiguration
      Checks if the queue participates in reservation based scheduling
      Specified by:
      isReservable in class ReservationSchedulerConfiguration
      Parameters:
      queue - name of the queue
      Returns:
      true if the queue participates in reservation based scheduling

    • setReservable

      public void setReservable(QueuePath queue, boolean isReservable)

    • getReservationWindow

      public long getReservationWindow(QueuePath queue)
      Description copied from class: ReservationSchedulerConfiguration
      Gets the length of time in milliseconds for which the SharingPolicy checks for validity
      Overrides:
      getReservationWindow in class ReservationSchedulerConfiguration
      Parameters:
      queue - name of the queue
      Returns:
      length in time in milliseconds for which to check the SharingPolicy

    • getAverageCapacity

      public float getAverageCapacity(QueuePath queue)
      Description copied from class: ReservationSchedulerConfiguration
      Gets the average allowed capacity which will aggregated over the ReservationSchedulerConfiguration.getReservationWindow(org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath) by the the SharingPolicy to check aggregate used capacity
      Overrides:
      getAverageCapacity in class ReservationSchedulerConfiguration
      Parameters:
      queue - name of the queue
      Returns:
      average capacity allowed by the SharingPolicy

    • getInstantaneousMaxCapacity

      public float getInstantaneousMaxCapacity(QueuePath queue)
      Description copied from class: ReservationSchedulerConfiguration
      Gets the maximum capacity at any time that the SharingPolicy allows
      Overrides:
      getInstantaneousMaxCapacity in class ReservationSchedulerConfiguration
      Parameters:
      queue - name of the queue
      Returns:
      maximum allowed capacity at any time

    • setInstantaneousMaxCapacity

      public void setInstantaneousMaxCapacity(QueuePath queue, float instMaxCapacity)

    • setReservationWindow

      public void setReservationWindow(QueuePath queue, long reservationWindow)

    • setAverageCapacity

      public void setAverageCapacity(QueuePath queue, float avgCapacity)

    • getReservationAdmissionPolicy

      public String getReservationAdmissionPolicy(QueuePath queue)
      Description copied from class: ReservationSchedulerConfiguration
      Gets the name of the SharingPolicy class associated with the queue
      Overrides:
      getReservationAdmissionPolicy in class ReservationSchedulerConfiguration
      Parameters:
      queue - name of the queue
      Returns:
      the class name of the SharingPolicy

    • setReservationAdmissionPolicy

      public void setReservationAdmissionPolicy(QueuePath queue, String reservationPolicy)

    • getReservationAgent

      public String getReservationAgent(QueuePath queue)
      Description copied from class: ReservationSchedulerConfiguration
      Gets the name of the ReservationAgent class associated with the queue
      Overrides:
      getReservationAgent in class ReservationSchedulerConfiguration
      Parameters:
      queue - name of the queue
      Returns:
      the class name of the ReservationAgent

    • setReservationAgent

      public void setReservationAgent(QueuePath queue, String reservationPolicy)

    • getShowReservationAsQueues

      public boolean getShowReservationAsQueues(QueuePath queuePath)
      Description copied from class: ReservationSchedulerConfiguration
      Checks whether the reservation queues be hidden or visible
      Overrides:
      getShowReservationAsQueues in class ReservationSchedulerConfiguration
      Parameters:
      queuePath - name of the queue
      Returns:
      true if reservation queues should be visible

    • getReplanner

      public String getReplanner(QueuePath queue)
      Description copied from class: ReservationSchedulerConfiguration
      Gets the name of the Planner class associated with the queue
      Overrides:
      getReplanner in class ReservationSchedulerConfiguration
      Parameters:
      queue - name of the queue
      Returns:
      the class name of the Planner

    • getMoveOnExpiry

      public boolean getMoveOnExpiry(QueuePath queue)
      Description copied from class: ReservationSchedulerConfiguration
      Gets whether the applications should be killed or moved to the parent queue when the ReservationDefinition expires
      Overrides:
      getMoveOnExpiry in class ReservationSchedulerConfiguration
      Parameters:
      queue - name of the queue
      Returns:
      true if application should be moved, false if they need to be killed

    • getEnforcementWindow

      public long getEnforcementWindow(QueuePath queue)
      Description copied from class: ReservationSchedulerConfiguration
      Gets the time in milliseconds for which the Planner will verify the Plans satisfy the constraints
      Overrides:
      getEnforcementWindow in class ReservationSchedulerConfiguration
      Parameters:
      queue - name of the queue
      Returns:
      the time in milliseconds for which to check constraints

    • setPreemptionDisabled

      public void setPreemptionDisabled(QueuePath queue, boolean preemptionDisabled)
      Sets the disable_preemption property in order to indicate whether or not container preemption will be disabled for the specified queue.
      Parameters:
      queue - queue path
      preemptionDisabled - true if preemption is disabled on queue

    • getPreemptionDisabled

      public boolean getPreemptionDisabled(QueuePath queue, boolean defaultVal)
      Indicates whether preemption is disabled on the specified queue.
      Parameters:
      queue - queue path to query
      defaultVal - used as default if the disable_preemption is not set in the configuration
      Returns:
      true if preemption is disabled on queue, false otherwise

    • getIntraQueuePreemptionDisabled

      public boolean getIntraQueuePreemptionDisabled(QueuePath queue, boolean defaultVal)
      Indicates whether intra-queue preemption is disabled on the specified queue
      Parameters:
      queue - queue path to query
      defaultVal - used as default if the property is not set in the configuration
      Returns:
      true if preemption is disabled on queue, false otherwise

    • setPreemptionObserveOnly

      public void setPreemptionObserveOnly(boolean value)

    • getPreemptionObserveOnly

      public boolean getPreemptionObserveOnly()

    • getConfiguredNodeLabels

      public Set<String> getConfiguredNodeLabels(QueuePath queuePath)
      Get configured node labels in a given queuePath.
      Parameters:
      queuePath - queue path.
      Returns:
      configured node labels.

    • getConfiguredNodeLabelsByQueue

      public Map<String,Set<String>> getConfiguredNodeLabelsByQueue()
      Get configured node labels for all queues that have accessible-node-labels prefixed properties set.
      Returns:
      configured node labels

    • getClusterLevelApplicationMaxPriority

      public org.apache.hadoop.yarn.api.records.Priority getClusterLevelApplicationMaxPriority()

    • getDefaultApplicationPriorityConfPerQueue

      public Integer getDefaultApplicationPriorityConfPerQueue(QueuePath queue)

    • setOrderingPolicy

      @VisibleForTesting public void setOrderingPolicy(QueuePath queue, String policy)

    • setOrderingPolicyParameter

      @VisibleForTesting public void setOrderingPolicyParameter(QueuePath queue, String parameterKey, String parameterValue)

    • getLazyPreemptionEnabled

      public boolean getLazyPreemptionEnabled()

    • shouldAppFailFast

      public static boolean shouldAppFailFast(org.apache.hadoop.conf.Configuration conf)

    • setDefaultMaxParallelApps

      public void setDefaultMaxParallelApps(int value)

    • getDefaultMaxParallelApps

      public Integer getDefaultMaxParallelApps()

    • setDefaultMaxParallelAppsPerUser

      public void setDefaultMaxParallelAppsPerUser(int value)

    • getDefaultMaxParallelAppsPerUser

      public Integer getDefaultMaxParallelAppsPerUser()

    • setMaxParallelAppsForUser

      public void setMaxParallelAppsForUser(String user, int value)

    • getMaxParallelAppsForUser

      public Integer getMaxParallelAppsForUser(String user)

    • setMaxParallelAppsForQueue

      public void setMaxParallelAppsForQueue(QueuePath queue, String value)

    • getMaxParallelAppsForQueue

      public Integer getMaxParallelAppsForQueue(QueuePath queue)

    • getAllowZeroCapacitySum

      public boolean getAllowZeroCapacitySum(QueuePath queue)

    • setAllowZeroCapacitySum

      public void setAllowZeroCapacitySum(QueuePath queue, boolean value)

    • getGlobalMaximumApplicationsPerQueue

      public int getGlobalMaximumApplicationsPerQueue()

    • setGlobalMaximumApplicationsPerQueue

      public void setGlobalMaximumApplicationsPerQueue(int val)

    • setQueueOrderingPolicy

      @Private public void setQueueOrderingPolicy(QueuePath queue, String policy)

    • getQueueOrderingPolicy

      @Private public QueueOrderingPolicy getQueueOrderingPolicy(QueuePath queue, String parentPolicy)

    • getPUOrderingPolicyUnderUtilizedPreemptionEnabled

      public boolean getPUOrderingPolicyUnderUtilizedPreemptionEnabled()
      Do we allow under-utilized queue with higher priority to preempt queue with lower priority *even if queue with lower priority is not satisfied*. For example, two queues, a and b a.priority = 1, (a.used-capacity - a.reserved-capacity) = 40% b.priority = 0, b.used-capacity = 30% Set this configuration to true to allow queue-a to preempt container from queue-b. (The reason why deduct reserved-capacity from used-capacity for queue with higher priority is: the reserved-capacity is just scheduler's internal implementation to allocate large containers, it is not possible for application to use such reserved-capacity. It is possible that a queue with large container requests have a large number of containers but cannot allocate from any of them. But scheduler will make sure a satisfied queue will not preempt resource from any other queues. A queue is considered to be satisfied when queue's used-capacity - reserved-capacity ≥ guaranteed-capacity.)
      Returns:
      allowed or not

    • setPUOrderingPolicyUnderUtilizedPreemptionEnabled

      @VisibleForTesting public void setPUOrderingPolicyUnderUtilizedPreemptionEnabled(boolean enabled)

    • getPUOrderingPolicyUnderUtilizedPreemptionDelay

      public long getPUOrderingPolicyUnderUtilizedPreemptionDelay()
      When a reserved container of an underutilized queue is created. Preemption will kick in after specified delay (in ms). The total time to preempt resources for a reserved container from higher priority queue will be: reserved-container-delay-ms + PREEMPTION_WAIT_TIME_BEFORE_KILL. This parameter is added to make preemption from lower priority queue which is underutilized to be more careful. This parameter takes effect when underutilized-preemption.enabled set to true.
      Returns:
      delay

    • setPUOrderingPolicyUnderUtilizedPreemptionDelay

      @VisibleForTesting public void setPUOrderingPolicyUnderUtilizedPreemptionDelay(long timeout)

    • getPUOrderingPolicyUnderUtilizedPreemptionMoveReservation

      public boolean getPUOrderingPolicyUnderUtilizedPreemptionMoveReservation()
      When doing preemption from under-satisfied queues for priority queue. Do we allow move reserved container from one host to another?
      Returns:
      allow or not

    • setPUOrderingPolicyUnderUtilizedPreemptionMoveReservation

      @VisibleForTesting public void setPUOrderingPolicyUnderUtilizedPreemptionMoveReservation(boolean allowMoveReservation)

    • getAllUserWeightsForQueue

      public UserWeights getAllUserWeightsForQueue(QueuePath queuePath)
      Get the weights of all users at this queue level from the configuration. Used in computing user-specific user limit, relative to other users.
      Parameters:
      queuePath - full queue path
      Returns:
      map of user weights, if they exist. Otherwise, return empty map.

    • getAssignMultipleEnabled

      public boolean getAssignMultipleEnabled()

    • getMaxAssignPerHeartbeat

      public int getMaxAssignPerHeartbeat()

    • getMaximumLifetimePerQueue

      public long getMaximumLifetimePerQueue(QueuePath queue)

    • setMaximumLifetimePerQueue

      public void setMaximumLifetimePerQueue(QueuePath queue, long maximumLifetime)

    • getDefaultLifetimePerQueue

      public long getDefaultLifetimePerQueue(QueuePath queue)

    • setDefaultLifetimePerQueue

      public void setDefaultLifetimePerQueue(QueuePath queue, long defaultLifetime)

    • isAutoCreateChildQueueEnabled

      @Private public boolean isAutoCreateChildQueueEnabled(QueuePath queuePath)
      If true, this queue will be created as a Parent Queue which Auto Created leaf child queues
      Parameters:
      queuePath - The queues path
      Returns:
      true if auto create is enabled for child queues else false. Default is false

    • setAutoCreateChildQueueEnabled

      @Private @VisibleForTesting public void setAutoCreateChildQueueEnabled(QueuePath queuePath, boolean autoCreationEnabled)

    • setAutoQueueCreationV2Enabled

      public void setAutoQueueCreationV2Enabled(QueuePath queuePath, boolean autoQueueCreation)

    • isAutoQueueCreationV2Enabled

      public boolean isAutoQueueCreationV2Enabled(QueuePath queuePath)

    • getShouldFailAutoQueueCreationWhenGuaranteedCapacityExceeded

      @Private public boolean getShouldFailAutoQueueCreationWhenGuaranteedCapacityExceeded(QueuePath queuePath)
      Fail further auto leaf queue creation when parent's guaranteed capacity is exceeded.
      Parameters:
      queuePath - the parent queue's path
      Returns:
      true if configured to fail else false

    • setShouldFailAutoQueueCreationWhenGuaranteedCapacityExceeded

      @VisibleForTesting @Private public void setShouldFailAutoQueueCreationWhenGuaranteedCapacityExceeded(QueuePath queuePath, boolean autoCreationEnabled)

    • getAutoCreatedQueuesMaxChildQueuesLimit

      @Private public int getAutoCreatedQueuesMaxChildQueuesLimit(QueuePath queuePath)
      Get the max number of leaf queues that are allowed to be created under a parent queue
      Parameters:
      queuePath - the paret queue's path
      Returns:
      the max number of leaf queues allowed to be auto created

    • getAutoCreatedQueuesV2MaxChildQueuesLimit

      @Private public int getAutoCreatedQueuesV2MaxChildQueuesLimit(QueuePath queuePath)
      Get the max number of queues that are allowed to be created under a parent queue which allowed auto creation v2.
      Parameters:
      queuePath - the parent queue's path
      Returns:
      the max number of queues allowed to be auto created, in new auto created.

    • setAutoCreatedQueuesV2MaxChildQueuesLimit

      @VisibleForTesting public void setAutoCreatedQueuesV2MaxChildQueuesLimit(QueuePath queuePath, int maxQueues)

    • isAutoExpiredDeletionEnabled

      @Private public boolean isAutoExpiredDeletionEnabled(QueuePath queuePath)
      If true, auto created queue with weight mode will be deleted when queue is expired.
      Parameters:
      queuePath - the queue's path for auto deletion check
      Returns:
      true if auto created queue's deletion when expired is enabled else false. Default is true.

    • setAutoExpiredDeletionEnabled

      @Private @VisibleForTesting public void setAutoExpiredDeletionEnabled(QueuePath queuePath, boolean autoRemovalEnable)

    • setAutoExpiredDeletionTime

      @Private @VisibleForTesting public void setAutoExpiredDeletionTime(long time)

    • getAutoExpiredDeletionTime

      @Private @VisibleForTesting public long getAutoExpiredDeletionTime()

    • getAutoCreatedQueueManagementPolicy

      @Private public String getAutoCreatedQueueManagementPolicy(QueuePath queue)
      Queue Management computation policy for Auto Created queues
      Parameters:
      queue - The queue's path
      Returns:
      Configured policy class name

    • getAutoCreatedQueueManagementPolicyClass

      @Private protected AutoCreatedQueueManagementPolicy getAutoCreatedQueueManagementPolicyClass(QueuePath queue)
      Get The policy class configured to manage capacities for auto created leaf queues under the specified parent
      Parameters:
      queue - The parent queue's path
      Returns:
      The policy class configured to manage capacities for auto created leaf queues under the specified parent queue

    • setAutoCreatedLeafQueueConfigCapacity

      @VisibleForTesting @Private public void setAutoCreatedLeafQueueConfigCapacity(QueuePath queuePath, float val)

    • setAutoCreatedLeafQueueTemplateCapacityByLabel

      @VisibleForTesting @Private public void setAutoCreatedLeafQueueTemplateCapacityByLabel(QueuePath queuePath, String label, float val)

    • setAutoCreatedLeafQueueTemplateCapacityByLabel

      @VisibleForTesting @Private public void setAutoCreatedLeafQueueTemplateCapacityByLabel(QueuePath queuePath, String label, org.apache.hadoop.yarn.api.records.Resource resource)

    • setAutoCreatedLeafQueueConfigMaxCapacity

      @Private @VisibleForTesting public void setAutoCreatedLeafQueueConfigMaxCapacity(QueuePath queuePath, float val)

    • setAutoCreatedLeafQueueTemplateMaxCapacity

      @Private @VisibleForTesting public void setAutoCreatedLeafQueueTemplateMaxCapacity(QueuePath queuePath, String label, float val)

    • setAutoCreatedLeafQueueTemplateMaxCapacity

      @Private @VisibleForTesting public void setAutoCreatedLeafQueueTemplateMaxCapacity(QueuePath queuePath, String label, org.apache.hadoop.yarn.api.records.Resource resource)

    • setAutoCreatedLeafQueueConfigUserLimit

      @VisibleForTesting @Private public void setAutoCreatedLeafQueueConfigUserLimit(QueuePath queuePath, int val)

    • setAutoCreatedLeafQueueConfigUserLimitFactor

      @VisibleForTesting @Private public void setAutoCreatedLeafQueueConfigUserLimitFactor(QueuePath queuePath, float val)

    • setAutoCreatedLeafQueueConfigDefaultNodeLabelExpression

      @Private @VisibleForTesting public void setAutoCreatedLeafQueueConfigDefaultNodeLabelExpression(QueuePath queuePath, String expression)

    • setAutoCreatedLeafQueueConfigMaximumAllocation

      @Private @VisibleForTesting public void setAutoCreatedLeafQueueConfigMaximumAllocation(QueuePath queuePath, String expression)

    • getUnits

      public static String getUnits(String resourceValue)

    • getMinimumResourceRequirement

      public org.apache.hadoop.yarn.api.records.Resource getMinimumResourceRequirement(String label, QueuePath queue, Set<String> resourceTypes)
      Get absolute minimum resource requirement for a queue.
      Parameters:
      label - NodeLabel
      queue - queue path
      resourceTypes - Resource types
      Returns:
      ResourceInformation

    • getMaximumResourceRequirement

      public org.apache.hadoop.yarn.api.records.Resource getMaximumResourceRequirement(String label, QueuePath queue, Set<String> resourceTypes)
      Get absolute maximum resource requirement for a queue.
      Parameters:
      label - NodeLabel
      queue - queue path
      resourceTypes - Resource types
      Returns:
      Resource

    • setMinimumResourceRequirement

      @VisibleForTesting public void setMinimumResourceRequirement(String label, QueuePath queue, org.apache.hadoop.yarn.api.records.Resource resource)

    • setMaximumResourceRequirement

      @VisibleForTesting public void setMaximumResourceRequirement(String label, QueuePath queue, org.apache.hadoop.yarn.api.records.Resource resource)

    • parseConfiguredResourceVector

      public Map<String,QueueCapacityVector> parseConfiguredResourceVector(QueuePath queuePath, Set<String> labels)

    • parseConfiguredMaximumCapacityVector

      public Map<String,QueueCapacityVector> parseConfiguredMaximumCapacityVector(QueuePath queuePath, Set<String> labels, QueueCapacityVector defaultVector)

    • checkConfigTypeIsAbsoluteResource

      public boolean checkConfigTypeIsAbsoluteResource(String label, QueuePath queue, Set<String> resourceTypes)

    • getMultiNodesSortingAlgorithmPolicy

      public String getMultiNodesSortingAlgorithmPolicy(QueuePath queue)

    • isLegacyQueueMode

      public boolean isLegacyQueueMode()

    • setLegacyQueueModeEnabled

      public void setLegacyQueueModeEnabled(boolean value)

    • getMultiNodePlacementEnabled

      public boolean getMultiNodePlacementEnabled()

    • getMultiNodePlacementPolicies

      public Set<MultiNodePolicySpec> getMultiNodePlacementPolicies()