-
Notifications
You must be signed in to change notification settings - Fork 1.8k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Integrated IO Based AdmissionController to AdmissionControl Framework
- Loading branch information
Ajay Kumar Movva
committed
Mar 11, 2024
1 parent
b15cb0c
commit e551547
Showing
12 changed files
with
999 additions
and
72 deletions.
There are no files selected for viewing
220 changes: 176 additions & 44 deletions
220
...rTest/java/org/opensearch/ratelimitting/admissioncontrol/AdmissionControlMultiNodeIT.java
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
132 changes: 132 additions & 0 deletions
132
...org/opensearch/ratelimitting/admissioncontrol/controllers/IoBasedAdmissionController.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,132 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.ratelimitting.admissioncontrol.controllers; | ||
|
|
||
| import org.apache.logging.log4j.LogManager; | ||
| import org.apache.logging.log4j.Logger; | ||
| import org.apache.lucene.util.Constants; | ||
| import org.opensearch.cluster.service.ClusterService; | ||
| import org.opensearch.common.settings.Settings; | ||
| import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; | ||
| import org.opensearch.node.NodeResourceUsageStats; | ||
| import org.opensearch.node.ResourceUsageCollectorService; | ||
| import org.opensearch.ratelimitting.admissioncontrol.enums.AdmissionControlActionType; | ||
| import org.opensearch.ratelimitting.admissioncontrol.enums.AdmissionControlMode; | ||
| import org.opensearch.ratelimitting.admissioncontrol.settings.IoBasedAdmissionControllerSettings; | ||
|
|
||
| import java.util.Locale; | ||
| import java.util.Optional; | ||
|
|
||
| public class IoBasedAdmissionController extends AdmissionController { | ||
| public static final String IO_BASED_ADMISSION_CONTROLLER = "global_io_usage"; | ||
| private static final Logger LOGGER = LogManager.getLogger(IoBasedAdmissionController.class); | ||
| public IoBasedAdmissionControllerSettings settings; | ||
|
|
||
| /** | ||
| * @param admissionControllerName name of the admissionController | ||
| * @param resourceUsageCollectorService instance used to get resource usage stats of the node | ||
| * @param clusterService instance of the clusterService | ||
| */ | ||
| public IoBasedAdmissionController( | ||
| String admissionControllerName, | ||
| ResourceUsageCollectorService resourceUsageCollectorService, | ||
| ClusterService clusterService, | ||
| Settings settings | ||
| ) { | ||
| super(admissionControllerName, resourceUsageCollectorService, clusterService); | ||
| this.settings = new IoBasedAdmissionControllerSettings(clusterService.getClusterSettings(), settings); | ||
| } | ||
|
|
||
| @Override | ||
| public boolean isEnabledForTransportLayer(AdmissionControlMode admissionControlMode) { | ||
| if (Constants.LINUX) { | ||
| return super.isEnabledForTransportLayer(admissionControlMode); | ||
| } | ||
| return false; | ||
| } | ||
|
|
||
| /** | ||
| * Apply admission control based on the resource usage for an action | ||
| * | ||
| * @param action is the transport action | ||
| * @param admissionControlActionType type of admissionControlActionType | ||
| */ | ||
| @Override | ||
| public void apply(String action, AdmissionControlActionType admissionControlActionType) { | ||
| if (this.isEnabledForTransportLayer(this.settings.getTransportLayerAdmissionControllerMode())) { | ||
| this.applyForTransportLayer(action, admissionControlActionType); | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Apply transport layer admission control if configured limit has been reached | ||
| */ | ||
| private void applyForTransportLayer(String actionName, AdmissionControlActionType admissionControlActionType) { | ||
| if (isLimitsBreached(actionName, admissionControlActionType)) { | ||
| this.addRejectionCount(admissionControlActionType.getType(), 1); | ||
| if (this.isAdmissionControllerEnforced(this.settings.getTransportLayerAdmissionControllerMode())) { | ||
| throw new OpenSearchRejectedExecutionException( | ||
| String.format( | ||
| Locale.ROOT, | ||
| "Io usage admission controller rejected the request for action [%s] as IO limit reached", | ||
| admissionControlActionType.name() | ||
| ) | ||
| ); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Check if the configured resource usage limits are breached for the action | ||
| */ | ||
| private boolean isLimitsBreached(String actionName, AdmissionControlActionType admissionControlActionType) { | ||
| // check if cluster state is ready | ||
| if (clusterService.state() != null && clusterService.state().nodes() != null) { | ||
| long maxIoLimit = this.getIoRejectionThreshold(admissionControlActionType); | ||
| Optional<NodeResourceUsageStats> nodePerformanceStatistics = this.resourceUsageCollectorService.getNodeStatistics( | ||
| this.clusterService.state().nodes().getLocalNodeId() | ||
| ); | ||
| if (nodePerformanceStatistics.isPresent()) { | ||
| double ioUsage = nodePerformanceStatistics.get().getIoUsageStats().getIoUtilisationPercent(); | ||
| if (ioUsage >= maxIoLimit) { | ||
| LOGGER.warn( | ||
| "IoBasedAdmissionController limit reached as the current IO " | ||
| + "usage [{}] exceeds the allowed limit [{}] for transport action [{}] in admissionControlMode [{}]", | ||
| ioUsage, | ||
| maxIoLimit, | ||
| actionName, | ||
| this.settings.getTransportLayerAdmissionControllerMode() | ||
| ); | ||
| return true; | ||
| } | ||
| } | ||
| } | ||
| return false; | ||
| } | ||
|
|
||
| /** | ||
| * Get IO rejection threshold based on action type | ||
| */ | ||
| private long getIoRejectionThreshold(AdmissionControlActionType admissionControlActionType) { | ||
| switch (admissionControlActionType) { | ||
| case SEARCH: | ||
| return this.settings.getSearchIOUsageLimit(); | ||
| case INDEXING: | ||
| return this.settings.getIndexingIOUsageLimit(); | ||
| default: | ||
| throw new IllegalArgumentException( | ||
| String.format( | ||
| Locale.ROOT, | ||
| "Admission control not Supported for AdmissionControlActionType: %s", | ||
| admissionControlActionType.getType() | ||
| ) | ||
| ); | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
98 changes: 98 additions & 0 deletions
98
...pensearch/ratelimitting/admissioncontrol/settings/IoBasedAdmissionControllerSettings.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,98 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.ratelimitting.admissioncontrol.settings; | ||
|
|
||
| import org.opensearch.common.settings.ClusterSettings; | ||
| import org.opensearch.common.settings.Setting; | ||
| import org.opensearch.common.settings.Settings; | ||
| import org.opensearch.ratelimitting.admissioncontrol.AdmissionControlSettings; | ||
| import org.opensearch.ratelimitting.admissioncontrol.enums.AdmissionControlMode; | ||
|
|
||
| /** | ||
| * Settings related to IO based admission controller. | ||
| * @opensearch.internal | ||
| */ | ||
| public class IoBasedAdmissionControllerSettings { | ||
|
|
||
| /** | ||
| * Default parameters for the IoBasedAdmissionControllerSettings | ||
| */ | ||
| public static class Defaults { | ||
| public static final long IO_USAGE_LIMIT = 95; | ||
| } | ||
|
|
||
| private AdmissionControlMode transportLayerMode; | ||
| private Long searchIOUsageLimit; | ||
| private Long indexingIOUsageLimit; | ||
|
|
||
| /** | ||
| * Feature level setting to operate in shadow-mode or in enforced-mode. If enforced field is set | ||
| * rejection will be performed, otherwise only rejection metrics will be populated. | ||
| */ | ||
| public static final Setting<AdmissionControlMode> IO_BASED_ADMISSION_CONTROLLER_TRANSPORT_LAYER_MODE = new Setting<>( | ||
| "admission_control.transport.io_usage.mode_override", | ||
| AdmissionControlSettings.ADMISSION_CONTROL_TRANSPORT_LAYER_MODE, | ||
| AdmissionControlMode::fromName, | ||
| Setting.Property.Dynamic, | ||
| Setting.Property.NodeScope | ||
| ); | ||
|
|
||
| /** | ||
| * This setting used to set the IO Limits for the search requests by default it will use default IO usage limit | ||
| */ | ||
| public static final Setting<Long> SEARCH_IO_USAGE_LIMIT = Setting.longSetting( | ||
| "admission_control.search.io_usage.limit", | ||
| Defaults.IO_USAGE_LIMIT, | ||
| Setting.Property.Dynamic, | ||
| Setting.Property.NodeScope | ||
| ); | ||
|
|
||
| /** | ||
| * This setting used to set the IO limits for the indexing requests by default it will use default IO usage limit | ||
| */ | ||
| public static final Setting<Long> INDEXING_IO_USAGE_LIMIT = Setting.longSetting( | ||
| "admission_control.indexing.io_usage.limit", | ||
| Defaults.IO_USAGE_LIMIT, | ||
| Setting.Property.Dynamic, | ||
| Setting.Property.NodeScope | ||
| ); | ||
|
|
||
| public IoBasedAdmissionControllerSettings(ClusterSettings clusterSettings, Settings settings) { | ||
| this.transportLayerMode = IO_BASED_ADMISSION_CONTROLLER_TRANSPORT_LAYER_MODE.get(settings); | ||
| clusterSettings.addSettingsUpdateConsumer(IO_BASED_ADMISSION_CONTROLLER_TRANSPORT_LAYER_MODE, this::setTransportLayerMode); | ||
| this.searchIOUsageLimit = SEARCH_IO_USAGE_LIMIT.get(settings); | ||
| this.indexingIOUsageLimit = INDEXING_IO_USAGE_LIMIT.get(settings); | ||
| clusterSettings.addSettingsUpdateConsumer(INDEXING_IO_USAGE_LIMIT, this::setIndexingIOUsageLimit); | ||
| clusterSettings.addSettingsUpdateConsumer(SEARCH_IO_USAGE_LIMIT, this::setSearchIOUsageLimit); | ||
| } | ||
|
|
||
| public void setIndexingIOUsageLimit(Long indexingIOUsageLimit) { | ||
| this.indexingIOUsageLimit = indexingIOUsageLimit; | ||
| } | ||
|
|
||
| public void setSearchIOUsageLimit(Long searchIOUsageLimit) { | ||
| this.searchIOUsageLimit = searchIOUsageLimit; | ||
| } | ||
|
|
||
| public AdmissionControlMode getTransportLayerAdmissionControllerMode() { | ||
| return transportLayerMode; | ||
| } | ||
|
|
||
| public void setTransportLayerMode(AdmissionControlMode transportLayerMode) { | ||
| this.transportLayerMode = transportLayerMode; | ||
| } | ||
|
|
||
| public Long getIndexingIOUsageLimit() { | ||
| return indexingIOUsageLimit; | ||
| } | ||
|
|
||
| public Long getSearchIOUsageLimit() { | ||
| return searchIOUsageLimit; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.