Skip to content

Commit be9afbc

Browse files
opensearch-trigger-bot[bot]jowg-amazonpetardzAngie Zhang
authored
Error prevention stage 1 (opensearch-project#579) (opensearch-project#591)
* Error prevention stage 1 * Removed recursion from Explain Action to avoid stackoverflow in some situations (opensearch-project#419) * enabled by default integrated * Fix Test cases * Fix comments; set validation disabled by default * Rename validation_service to action_validation; Fix some detekt issues Signed-off-by: Joanne Wang <jowg@amazon.com> Signed-off-by: Petar Dzepina <petar.dzepina@gmail.com> Signed-off-by: Angie Zhang <langelzh@amazon.com> Co-authored-by: Joanne Wang <jowg@amazon.com> Co-authored-by: Petar <petar.dzepina@gmail.com> Co-authored-by: Angie Zhang <langelzh@amazon.com>
1 parent 2c8160a commit be9afbc

File tree

60 files changed

+1904
-50
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

60 files changed

+1904
-50
lines changed

‎spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/Validate.kt‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package org.opensearch.indexmanagement.spi.indexstatemanagement
7+
8+
import org.opensearch.cluster.service.ClusterService
9+
import org.opensearch.common.io.stream.StreamInput
10+
import org.opensearch.common.io.stream.StreamOutput
11+
import org.opensearch.common.io.stream.Writeable
12+
import org.opensearch.common.settings.Settings
13+
import org.opensearch.monitor.jvm.JvmService
14+
import java.util.Locale
15+
16+
abstract class Validate(
17+
val settings: Settings,
18+
val clusterService: ClusterService,
19+
val jvmService: JvmService
20+
) {
21+
22+
var validationStatus = ValidationStatus.PASSED
23+
var validationMessage: String? = "Starting Validation"
24+
25+
abstract fun execute(indexName: String): Validate
26+
27+
enum class ValidationStatus(val status: String) : Writeable {
28+
PASSED("passed"),
29+
RE_VALIDATING("re_validating"),
30+
FAILED("failed");
31+
32+
override fun toString(): String {
33+
return status
34+
}
35+
36+
override fun writeTo(out: StreamOutput) {
37+
out.writeString(status)
38+
}
39+
40+
companion object {
41+
fun read(streamInput: StreamInput): ValidationStatus {
42+
return valueOf(streamInput.readString().uppercase(Locale.ROOT))
43+
}
44+
}
45+
}
46+
}

‎spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/model/ManagedIndexMetaData.kt‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -241,7 +241,6 @@ data class ManagedIndexMetaData(
241241
var action: ActionMetaData? = null
242242
var step: StepMetaData? = null
243243
var retryInfo: PolicyRetryInfoMetaData? = null
244-
245244
var info: Map<String, Any>? = null
246245

247246
XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp)

‎spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/model/ValidationResult.kt‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package org.opensearch.indexmanagement.spi.indexstatemanagement.model
7+
8+
import org.opensearch.common.Strings
9+
import org.opensearch.common.io.stream.StreamInput
10+
import org.opensearch.common.io.stream.StreamOutput
11+
import org.opensearch.common.io.stream.Writeable
12+
import org.opensearch.common.xcontent.LoggingDeprecationHandler
13+
import org.opensearch.common.xcontent.NamedXContentRegistry
14+
import org.opensearch.common.xcontent.ToXContent
15+
import org.opensearch.common.xcontent.ToXContentFragment
16+
import org.opensearch.common.xcontent.XContentBuilder
17+
import org.opensearch.common.xcontent.XContentParser
18+
import org.opensearch.common.xcontent.XContentParserUtils
19+
import org.opensearch.common.xcontent.XContentType
20+
import org.opensearch.indexmanagement.spi.indexstatemanagement.Validate
21+
import java.io.ByteArrayInputStream
22+
import java.nio.charset.StandardCharsets
23+
import java.util.Locale
24+
25+
data class ValidationResult(
26+
val validationMessage: String,
27+
val validationStatus: Validate.ValidationStatus
28+
) : Writeable, ToXContentFragment {
29+
30+
override fun writeTo(out: StreamOutput) {
31+
out.writeString(validationMessage)
32+
validationStatus.writeTo(out)
33+
}
34+
35+
override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder {
36+
builder
37+
.field(VALIDATION_MESSAGE, validationMessage)
38+
.field(VALIDATION_STATUS, validationStatus.toString())
39+
return builder
40+
}
41+
42+
fun getMapValueString(): String {
43+
return Strings.toString(this, false, false)
44+
}
45+
46+
companion object {
47+
const val VALIDATE = "validate"
48+
const val VALIDATION_MESSAGE = "validation_message"
49+
const val VALIDATION_STATUS = "validation_status"
50+
51+
fun fromStreamInput(si: StreamInput): ValidationResult {
52+
val validationMessage: String? = si.readString()
53+
val validationStatus: Validate.ValidationStatus? = Validate.ValidationStatus.read(si)
54+
55+
return ValidationResult(
56+
requireNotNull(validationMessage) { "$VALIDATION_MESSAGE is null" },
57+
requireNotNull(validationStatus) { "$VALIDATION_STATUS is null" }
58+
)
59+
}
60+
61+
fun fromManagedIndexMetaDataMap(map: Map<String, String?>): ValidationResult? {
62+
val stepJsonString = map[VALIDATE]
63+
return if (stepJsonString != null) {
64+
val inputStream = ByteArrayInputStream(stepJsonString.toByteArray(StandardCharsets.UTF_8))
65+
val parser = XContentType.JSON.xContent().createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, inputStream)
66+
parser.nextToken()
67+
parse(parser)
68+
} else {
69+
null
70+
}
71+
}
72+
73+
fun parse(xcp: XContentParser): ValidationResult {
74+
var validationMessage: String? = null
75+
var validationStatus: Validate.ValidationStatus? = null
76+
77+
XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp)
78+
while (xcp.nextToken() != XContentParser.Token.END_OBJECT) {
79+
val fieldName = xcp.currentName()
80+
xcp.nextToken()
81+
82+
when (fieldName) {
83+
VALIDATION_MESSAGE -> validationMessage = xcp.text()
84+
VALIDATION_STATUS -> validationStatus = Validate.ValidationStatus.valueOf(xcp.text().uppercase(Locale.ROOT))
85+
}
86+
}
87+
88+
return ValidationResult(
89+
requireNotNull(validationMessage) { "$VALIDATION_MESSAGE is null" },
90+
requireNotNull(validationStatus) { "$VALIDATION_STATUS is null" }
91+
)
92+
}
93+
}
94+
}

‎src/main/kotlin/org/opensearch/indexmanagement/IndexManagementPlugin.kt‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,7 @@ import org.opensearch.indexmanagement.indexstatemanagement.transport.action.retr
7474
import org.opensearch.indexmanagement.indexstatemanagement.transport.action.updateindexmetadata.TransportUpdateManagedIndexMetaDataAction
7575
import org.opensearch.indexmanagement.indexstatemanagement.transport.action.updateindexmetadata.UpdateManagedIndexMetaDataAction
7676
import org.opensearch.indexmanagement.indexstatemanagement.util.DEFAULT_INDEX_TYPE
77+
import org.opensearch.indexmanagement.indexstatemanagement.validation.ActionValidation
7778
import org.opensearch.indexmanagement.indexstatemanagement.migration.ISMTemplateService
7879
import org.opensearch.indexmanagement.refreshanalyzer.RefreshSearchAnalyzerAction
7980
import org.opensearch.indexmanagement.refreshanalyzer.RestRefreshSearchAnalyzerAction
@@ -187,6 +188,7 @@ class IndexManagementPlugin : JobSchedulerExtension, NetworkPlugin, ActionPlugin
187188

188189
private val logger = LogManager.getLogger(javaClass)
189190
lateinit var indexManagementIndices: IndexManagementIndices
191+
lateinit var actionValidation: ActionValidation
190192
lateinit var clusterService: ClusterService
191193
lateinit var indexNameExpressionResolver: IndexNameExpressionResolver
192194
lateinit var rollupInterceptor: RollupInterceptor
@@ -387,6 +389,7 @@ class IndexManagementPlugin : JobSchedulerExtension, NetworkPlugin, ActionPlugin
387389
.registerConsumers()
388390
.registerClusterConfigurationProvider(skipFlag)
389391
indexManagementIndices = IndexManagementIndices(settings, client.admin().indices(), clusterService)
392+
actionValidation = ActionValidation(settings, clusterService, jvmService)
390393
val indexStateManagementHistory =
391394
IndexStateManagementHistory(
392395
settings,
@@ -408,6 +411,7 @@ class IndexManagementPlugin : JobSchedulerExtension, NetworkPlugin, ActionPlugin
408411
val managedIndexRunner = ManagedIndexRunner
409412
.registerClient(client)
410413
.registerClusterService(clusterService)
414+
.registerValidationService(actionValidation)
411415
.registerNamedXContentRegistry(xContentRegistry)
412416
.registerScriptService(scriptService)
413417
.registerSettings(settings)
@@ -436,6 +440,7 @@ class IndexManagementPlugin : JobSchedulerExtension, NetworkPlugin, ActionPlugin
436440
rollupRunner,
437441
transformRunner,
438442
indexManagementIndices,
443+
actionValidation,
439444
managedIndexCoordinator,
440445
indexStateManagementHistory,
441446
indexMetadataProvider,
@@ -458,6 +463,7 @@ class IndexManagementPlugin : JobSchedulerExtension, NetworkPlugin, ActionPlugin
458463
ManagedIndexSettings.ROLLOVER_ALIAS,
459464
ManagedIndexSettings.ROLLOVER_SKIP,
460465
ManagedIndexSettings.INDEX_STATE_MANAGEMENT_ENABLED,
466+
ManagedIndexSettings.ACTION_VALIDATION_ENABLED,
461467
ManagedIndexSettings.METADATA_SERVICE_ENABLED,
462468
ManagedIndexSettings.AUTO_MANAGE,
463469
ManagedIndexSettings.METADATA_SERVICE_STATUS,

‎src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/ManagedIndexRunner.kt‎

Lines changed: 32 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -51,8 +51,10 @@ import org.opensearch.indexmanagement.indexstatemanagement.settings.ManagedIndex
5151
import org.opensearch.indexmanagement.indexstatemanagement.settings.ManagedIndexSettings.Companion.ALLOW_LIST_NONE
5252
import org.opensearch.indexmanagement.indexstatemanagement.settings.ManagedIndexSettings.Companion.DEFAULT_ISM_ENABLED
5353
import org.opensearch.indexmanagement.indexstatemanagement.settings.ManagedIndexSettings.Companion.DEFAULT_JOB_INTERVAL
54+
import org.opensearch.indexmanagement.indexstatemanagement.settings.ManagedIndexSettings.Companion.DEFAULT_ACTION_VALIDATION_ENABLED
5455
import org.opensearch.indexmanagement.indexstatemanagement.settings.ManagedIndexSettings.Companion.INDEX_STATE_MANAGEMENT_ENABLED
5556
import org.opensearch.indexmanagement.indexstatemanagement.settings.ManagedIndexSettings.Companion.JOB_INTERVAL
57+
import org.opensearch.indexmanagement.indexstatemanagement.settings.ManagedIndexSettings.Companion.ACTION_VALIDATION_ENABLED
5658
import org.opensearch.indexmanagement.indexstatemanagement.util.DEFAULT_INDEX_TYPE
5759
import org.opensearch.indexmanagement.indexstatemanagement.util.MetadataCheck
5860
import org.opensearch.indexmanagement.indexstatemanagement.util.checkMetadata
@@ -74,6 +76,7 @@ import org.opensearch.indexmanagement.indexstatemanagement.util.sendNotification
7476
import org.opensearch.indexmanagement.indexstatemanagement.util.shouldBackoff
7577
import org.opensearch.indexmanagement.indexstatemanagement.util.shouldChangePolicy
7678
import org.opensearch.indexmanagement.indexstatemanagement.util.updateDisableManagedIndexRequest
79+
import org.opensearch.indexmanagement.indexstatemanagement.validation.ActionValidation
7780
import org.opensearch.indexmanagement.opensearchapi.IndexManagementSecurityContext
7881
import org.opensearch.indexmanagement.opensearchapi.convertToMap
7982
import org.opensearch.indexmanagement.opensearchapi.parseWithType
@@ -83,6 +86,7 @@ import org.opensearch.indexmanagement.opensearchapi.suspendUntil
8386
import org.opensearch.indexmanagement.opensearchapi.withClosableContext
8487
import org.opensearch.indexmanagement.spi.indexstatemanagement.Action
8588
import org.opensearch.indexmanagement.spi.indexstatemanagement.Step
89+
import org.opensearch.indexmanagement.spi.indexstatemanagement.Validate
8690
import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ActionMetaData
8791
import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData
8892
import org.opensearch.indexmanagement.spi.indexstatemanagement.model.PolicyRetryInfoMetaData
@@ -114,12 +118,14 @@ object ManagedIndexRunner :
114118
private lateinit var scriptService: ScriptService
115119
private lateinit var settings: Settings
116120
private lateinit var imIndices: IndexManagementIndices
121+
lateinit var actionValidation: ActionValidation
117122
private lateinit var ismHistory: IndexStateManagementHistory
118123
private lateinit var skipExecFlag: SkipExecution
119124
private lateinit var threadPool: ThreadPool
120125
private lateinit var extensionStatusChecker: ExtensionStatusChecker
121126
private lateinit var indexMetadataProvider: IndexMetadataProvider
122127
private var indexStateManagementEnabled: Boolean = DEFAULT_ISM_ENABLED
128+
private var validationServiceEnabled: Boolean = DEFAULT_ACTION_VALIDATION_ENABLED
123129
@Suppress("MagicNumber")
124130
private val savePolicyRetryPolicy = BackoffPolicy.exponentialBackoff(TimeValue.timeValueMillis(250), 3)
125131
@Suppress("MagicNumber")
@@ -166,6 +172,11 @@ object ManagedIndexRunner :
166172
indexStateManagementEnabled = it
167173
}
168174

175+
validationServiceEnabled = ACTION_VALIDATION_ENABLED.get(settings)
176+
clusterService.clusterSettings.addSettingsUpdateConsumer(ACTION_VALIDATION_ENABLED) {
177+
validationServiceEnabled = it
178+
}
179+
169180
allowList = ALLOW_LIST.get(settings)
170181
clusterService.clusterSettings.addSettingsUpdateConsumer(ALLOW_LIST) {
171182
allowList = it
@@ -179,6 +190,11 @@ object ManagedIndexRunner :
179190
return this
180191
}
181192

193+
fun registerValidationService(actionValidation: ActionValidation): ManagedIndexRunner {
194+
this.actionValidation = actionValidation
195+
return this
196+
}
197+
182198
fun registerHistoryIndex(ismHistory: IndexStateManagementHistory): ManagedIndexRunner {
183199
this.ismHistory = ismHistory
184200
return this
@@ -385,8 +401,23 @@ object ManagedIndexRunner :
385401
val startingManagedIndexMetaData = managedIndexMetaData.getStartingManagedIndexMetaData(state, action, step)
386402
val updateResult = updateManagedIndexMetaData(startingManagedIndexMetaData)
387403

388-
@Suppress("ComplexCondition")
404+
@Suppress("ComplexCondition", "MaxLineLength")
389405
if (updateResult.metadataSaved && state != null && action != null && step != null && currentActionMetaData != null) {
406+
if (validationServiceEnabled) {
407+
val validationResult = actionValidation.validate(action.type, stepContext.metadata.index)
408+
if (validationResult.validationStatus == Validate.ValidationStatus.RE_VALIDATING) {
409+
logger.warn("Validation Status is: RE_VALIDATING. The action is {}, state is {}, step is {}.\", action.type, state.name, step.name")
410+
publishErrorNotification(policy, managedIndexMetaData)
411+
return
412+
}
413+
if (validationResult.validationStatus == Validate.ValidationStatus.FAILED) {
414+
logger.warn("Validation Status is: FAILED. The action is {}, state is {}, step is {}.", action.type, state.name, step.name)
415+
publishErrorNotification(policy, managedIndexMetaData)
416+
disableManagedIndexConfig(managedIndexConfig)
417+
return
418+
}
419+
}
420+
390421
// Step null check is done in getStartingManagedIndexMetaData
391422
withClosableContext(
392423
IndexManagementSecurityContext(
@@ -671,7 +702,6 @@ object ManagedIndexRunner :
671702
} catch (e: Exception) {
672703
logger.error("Failed to save ManagedIndexMetaData for [index=${managedIndexMetaData.index}]", e)
673704
}
674-
675705
return result
676706
}
677707

‎src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestExplainAction.kt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,9 @@ import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.ISM_BASE_U
1313
import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.LEGACY_ISM_BASE_URI
1414
import org.opensearch.indexmanagement.indexstatemanagement.transport.action.explain.ExplainAction
1515
import org.opensearch.indexmanagement.indexstatemanagement.transport.action.explain.ExplainRequest
16+
import org.opensearch.indexmanagement.indexstatemanagement.util.DEFAULT_EXPLAIN_VALIDATE_ACTION
1617
import org.opensearch.indexmanagement.indexstatemanagement.util.DEFAULT_EXPLAIN_SHOW_POLICY
18+
import org.opensearch.indexmanagement.indexstatemanagement.util.SHOW_VALIDATE_ACTION
1719
import org.opensearch.indexmanagement.indexstatemanagement.util.DEFAULT_INDEX_TYPE
1820
import org.opensearch.indexmanagement.indexstatemanagement.util.DEFAULT_JOB_SORT_FIELD
1921
import org.opensearch.indexmanagement.indexstatemanagement.util.SHOW_POLICY_QUERY_PARAM
@@ -77,6 +79,7 @@ class RestExplainAction : BaseRestHandler() {
7779
clusterManagerTimeout,
7880
searchParams,
7981
request.paramAsBoolean(SHOW_POLICY_QUERY_PARAM, DEFAULT_EXPLAIN_SHOW_POLICY),
82+
request.paramAsBoolean(SHOW_VALIDATE_ACTION, DEFAULT_EXPLAIN_VALIDATE_ACTION),
8083
indexType
8184
)
8285

‎src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/settings/ManagedIndexSettings.kt‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import java.util.function.Function
1414
class ManagedIndexSettings {
1515
companion object {
1616
const val DEFAULT_ISM_ENABLED = true
17+
const val DEFAULT_ACTION_VALIDATION_ENABLED = false
1718
const val DEFAULT_TEMPLATE_MIGRATION_TIMESTAMP = 0L
1819
const val DEFAULT_JOB_INTERVAL = 5
1920
const val DEFAULT_JITTER = 0.6
@@ -28,6 +29,13 @@ class ManagedIndexSettings {
2829
Setting.Property.Dynamic
2930
)
3031

32+
val ACTION_VALIDATION_ENABLED: Setting<Boolean> = Setting.boolSetting(
33+
"plugins.index_state_management.action_validation.enabled",
34+
DEFAULT_ACTION_VALIDATION_ENABLED,
35+
Setting.Property.NodeScope,
36+
Setting.Property.Dynamic
37+
)
38+
3139
// 0: migration is going on
3240
// 1: migration succeed
3341
// -1: migration failed

‎src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/ExplainRequest.kt‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,21 +22,25 @@ class ExplainRequest : ActionRequest {
2222
val clusterManagerTimeout: TimeValue
2323
val searchParams: SearchParams
2424
val showPolicy: Boolean
25+
val validateAction: Boolean
2526
val indexType: String
2627

28+
@Suppress("LongParameterList")
2729
constructor(
2830
indices: List<String>,
2931
local: Boolean,
3032
clusterManagerTimeout: TimeValue,
3133
searchParams: SearchParams,
3234
showPolicy: Boolean,
35+
validateAction: Boolean,
3336
indexType: String
3437
) : super() {
3538
this.indices = indices
3639
this.local = local
3740
this.clusterManagerTimeout = clusterManagerTimeout
3841
this.searchParams = searchParams
3942
this.showPolicy = showPolicy
43+
this.validateAction = validateAction
4044
this.indexType = indexType
4145
}
4246

@@ -47,6 +51,7 @@ class ExplainRequest : ActionRequest {
4751
clusterManagerTimeout = sin.readTimeValue(),
4852
searchParams = SearchParams(sin),
4953
showPolicy = sin.readBoolean(),
54+
validateAction = sin.readBoolean(),
5055
indexType = sin.readString()
5156
)
5257

@@ -68,6 +73,7 @@ class ExplainRequest : ActionRequest {
6873
out.writeTimeValue(clusterManagerTimeout)
6974
searchParams.writeTo(out)
7075
out.writeBoolean(showPolicy)
76+
out.writeBoolean(validateAction)
7177
out.writeString(indexType)
7278
}
7379

0 commit comments

Comments
 (0)