Skip to content

Commit b72eeaa

Browse files
author
Angie Zhang
authored
Force using snakeyaml version 1.32 to fix CVE issue (#535)
* Force using snakeyaml version 1.32 to fix CVE issue Signed-off-by: Angie Zhang <langelzh@amazon.com> * Force using snakeyaml version 1.32 to fix CVE issue Signed-off-by: Angie Zhang <langelzh@amazon.com> * Force using snakeyaml version 1.32 to fix CVE issue Signed-off-by: Angie Zhang <langelzh@amazon.com> * Force using snakeyaml version 1.32 to fix CVE issue Signed-off-by: Angie Zhang <langelzh@amazon.com> * Kotlin version upgrade compatibility and jackson version upgrade Signed-off-by: Angie Zhang <langelzh@amazon.com> * Kotlin version upgrade compatibility Signed-off-by: Angie Zhang <langelzh@amazon.com> * detekt error fixing Signed-off-by: Angie Zhang <langelzh@amazon.com> * Update detekt setting Signed-off-by: Angie Zhang <langelzh@amazon.com> * Update detekt setting Signed-off-by: Angie Zhang <langelzh@amazon.com> * Update gradle typo Signed-off-by: Angie Zhang <langelzh@amazon.com> * Fix ktlint Signed-off-by: Angie Zhang <langelzh@amazon.com> * Update gradle Signed-off-by: Angie Zhang <langelzh@amazon.com> * Update gradle Signed-off-by: Angie Zhang <langelzh@amazon.com> * Fix build.gradle Signed-off-by: Angie Zhang <langelzh@amazon.com> * Fix jacoco tool version Signed-off-by: Angie Zhang <langelzh@amazon.com> Signed-off-by: Angie Zhang <langelzh@amazon.com>
1 parent 73161b5 commit b72eeaa

File tree

25 files changed

+151
-92
lines changed

25 files changed

+151
-92
lines changed

‎build.gradle‎

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ buildscript {
1919
notification_version = System.getProperty("notification.version", opensearch_build)
2020
common_utils_version = System.getProperty("common_utils.version", opensearch_build)
2121
job_scheduler_version = System.getProperty("job_scheduler_version.version", opensearch_build)
22-
kotlin_version = System.getProperty("kotlin.version", "1.4.32")
22+
kotlin_version = System.getProperty("kotlin.version", "1.6.10")
2323

2424
opensearch_no_snapshot = opensearch_version.replace("-SNAPSHOT","")
2525
job_scheduler_no_snapshot = job_scheduler_version.replace("-SNAPSHOT","")
@@ -39,16 +39,19 @@ buildscript {
3939
classpath "org.opensearch.gradle:build-tools:${opensearch_version}"
4040
classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:${kotlin_version}"
4141
classpath "org.jetbrains.kotlin:kotlin-allopen:${kotlin_version}"
42-
classpath "io.gitlab.arturbosch.detekt:detekt-gradle-plugin:1.17.1"
43-
classpath "org.jacoco:org.jacoco.agent:0.8.5"
42+
classpath "io.gitlab.arturbosch.detekt:detekt-gradle-plugin:1.21.0"
43+
classpath "org.jacoco:org.jacoco.agent:0.8.7"
4444
}
4545
}
4646

47+
48+
4749
plugins {
4850
id 'nebula.ospackage' version "8.3.0"
4951
id "com.dorongold.task-tree" version "1.5"
5052
}
5153

54+
5255
apply plugin: 'java'
5356
apply plugin: 'jacoco'
5457
apply plugin: 'idea'
@@ -73,8 +76,8 @@ configurations.all {
7376
force 'org.apache.httpcomponents:httpclient-osgi:4.5.13'
7477
force 'org.apache.httpcomponents.client5:httpclient5:5.0.3'
7578
force 'org.apache.httpcomponents.client5:httpclient5-osgi:5.0.3'
76-
force 'com.fasterxml.jackson.core:jackson-databind:2.10.4'
77-
force 'org.yaml:snakeyaml:1.26'
79+
force 'com.fasterxml.jackson.core:jackson-databind:2.13.4'
80+
force 'org.yaml:snakeyaml:1.32'
7881
force 'org.codehaus.plexus:plexus-utils:3.0.24'
7982
}
8083
}
@@ -127,7 +130,7 @@ detekt {
127130
buildUponDefaultConfig = true
128131
}
129132

130-
configurations.testCompile {
133+
configurations.testImplementation {
131134
exclude module: "securemock"
132135
}
133136

@@ -144,6 +147,7 @@ allprojects {
144147
if (isSnapshot) {
145148
version += "-SNAPSHOT"
146149
}
150+
jacoco.toolVersion = "0.8.7"
147151
}
148152

149153
dependencies {
@@ -164,7 +168,7 @@ dependencies {
164168
testImplementation "com.nhaarman.mockitokotlin2:mockito-kotlin:2.2.0"
165169
testCompile "org.mockito:mockito-core:3.12.4"
166170

167-
add("ktlint", "com.pinterest:ktlint:0.41.0") {
171+
add("ktlint", "com.pinterest:ktlint:0.45.1") {
168172
attributes {
169173
attribute(Bundling.BUNDLING_ATTRIBUTE, objects.named(Bundling, Bundling.EXTERNAL))
170174
}
@@ -602,4 +606,4 @@ task updateVersion {
602606
// String tokenization to support -SNAPSHOT
603607
ant.replaceregexp(file:'build.gradle', match: '"opensearch.version", "\\d.*"', replace: '"opensearch.version", "' + newVersion.tokenize('-')[0] + '-SNAPSHOT"', flags:'g', byline:true)
604608
}
605-
}
609+
}

‎detekt.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# TODO: Remove this before initial release, only for developmental purposes
22
build:
3-
maxIssues: 10
3+
maxIssues: 20
44

55
exceptions:
66
TooGenericExceptionCaught:

‎spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/Step.kt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ abstract class Step(val name: String, val isSafeToDisableOn: Boolean = true) {
6868

6969
companion object {
7070
fun read(streamInput: StreamInput): StepStatus {
71-
return valueOf(streamInput.readString().toUpperCase(Locale.ROOT))
71+
return valueOf(streamInput.readString().uppercase(Locale.ROOT))
7272
}
7373
}
7474
}

‎spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/model/ActionRetry.kt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ data class ActionRetry(
7272

7373
when (fieldName) {
7474
COUNT_FIELD -> count = xcp.longValue()
75-
BACKOFF_FIELD -> backoff = Backoff.valueOf(xcp.text().toUpperCase(Locale.ROOT))
75+
BACKOFF_FIELD -> backoff = Backoff.valueOf(xcp.text().uppercase(Locale.ROOT))
7676
DELAY_FIELD -> delay = TimeValue.parseTimeValue(xcp.text(), DELAY_FIELD)
7777
}
7878
}

‎spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/model/StepMetaData.kt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@ data class StepMetaData(
9090
when (fieldName) {
9191
NAME -> name = xcp.text()
9292
START_TIME -> startTime = xcp.longValue()
93-
STEP_STATUS -> stepStatus = Step.StepStatus.valueOf(xcp.text().toUpperCase(Locale.ROOT))
93+
STEP_STATUS -> stepStatus = Step.StepStatus.valueOf(xcp.text().uppercase(Locale.ROOT))
9494
}
9595
}
9696

‎src/main/kotlin/org/opensearch/indexmanagement/IndexManagementPlugin.kt‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44
*/
55

66
package org.opensearch.indexmanagement
7-
87
import org.apache.logging.log4j.LogManager
98
import org.opensearch.action.ActionRequest
109
import org.opensearch.action.ActionResponse
@@ -38,6 +37,7 @@ import org.opensearch.indexmanagement.indexstatemanagement.ManagedIndexCoordinat
3837
import org.opensearch.indexmanagement.indexstatemanagement.ManagedIndexRunner
3938
import org.opensearch.indexmanagement.indexstatemanagement.MetadataService
4039
import org.opensearch.indexmanagement.indexstatemanagement.SkipExecution
40+
import org.opensearch.indexmanagement.indexstatemanagement.migration.ISMTemplateService
4141
import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexConfig
4242
import org.opensearch.indexmanagement.indexstatemanagement.model.Policy
4343
import org.opensearch.indexmanagement.indexstatemanagement.resthandler.RestAddPolicyAction
@@ -73,7 +73,6 @@ import org.opensearch.indexmanagement.indexstatemanagement.transport.action.retr
7373
import org.opensearch.indexmanagement.indexstatemanagement.transport.action.updateindexmetadata.TransportUpdateManagedIndexMetaDataAction
7474
import org.opensearch.indexmanagement.indexstatemanagement.transport.action.updateindexmetadata.UpdateManagedIndexMetaDataAction
7575
import org.opensearch.indexmanagement.indexstatemanagement.util.DEFAULT_INDEX_TYPE
76-
import org.opensearch.indexmanagement.migration.ISMTemplateService
7776
import org.opensearch.indexmanagement.refreshanalyzer.RefreshSearchAnalyzerAction
7877
import org.opensearch.indexmanagement.refreshanalyzer.RestRefreshSearchAnalyzerAction
7978
import org.opensearch.indexmanagement.refreshanalyzer.TransportRefreshSearchAnalyzerAction
@@ -251,15 +250,15 @@ class IndexManagementPlugin : JobSchedulerExtension, NetworkPlugin, ActionPlugin
251250
indexManagementExtensions.forEach { extension ->
252251
val extensionName = extension.getExtensionName()
253252
if (extensionName in extensions) {
254-
throw IllegalStateException("Multiple extensions of IndexManagement have same name $extensionName - not supported")
253+
error("Multiple extensions of IndexManagement have same name $extensionName - not supported")
255254
}
256255
extension.getISMActionParsers().forEach { parser ->
257256
ISMActionsParser.instance.addParser(parser, extensionName)
258257
}
259258
indexMetadataServices.add(extension.getIndexMetadataService())
260259
extension.overrideClusterStateIndexUuidSetting()?.let {
261260
if (customIndexUUIDSetting != null) {
262-
throw IllegalStateException(
261+
error(
263262
"Multiple extensions of IndexManagement plugin overriding ClusterStateIndexUUIDSetting - not supported"
264263
)
265264
}
@@ -359,7 +358,9 @@ class IndexManagementPlugin : JobSchedulerExtension, NetworkPlugin, ActionPlugin
359358
)
360359

361360
indexMetadataProvider = IndexMetadataProvider(
362-
settings, client, clusterService,
361+
settings,
362+
client,
363+
clusterService,
363364
hashMapOf(
364365
DEFAULT_INDEX_TYPE to DefaultIndexMetadataService(customIndexUUIDSetting)
365366
)
@@ -386,7 +387,13 @@ class IndexManagementPlugin : JobSchedulerExtension, NetworkPlugin, ActionPlugin
386387

387388
val managedIndexCoordinator = ManagedIndexCoordinator(
388389
environment.settings(),
389-
client, clusterService, threadPool, indexManagementIndices, metadataService, templateService, indexMetadataProvider
390+
client,
391+
clusterService,
392+
threadPool,
393+
indexManagementIndices,
394+
metadataService,
395+
templateService,
396+
indexMetadataProvider
390397
)
391398

392399
return listOf(

‎src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/ManagedIndexCoordinator.kt‎

Lines changed: 21 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ import org.opensearch.index.query.QueryBuilders
4747
import org.opensearch.indexmanagement.IndexManagementIndices
4848
import org.opensearch.indexmanagement.IndexManagementPlugin
4949
import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.INDEX_MANAGEMENT_INDEX
50+
import org.opensearch.indexmanagement.indexstatemanagement.migration.ISMTemplateService
5051
import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexConfig
5152
import org.opensearch.indexmanagement.indexstatemanagement.model.Policy
5253
import org.opensearch.indexmanagement.indexstatemanagement.model.coordinator.ClusterStateManagedIndexConfig
@@ -73,7 +74,6 @@ import org.opensearch.indexmanagement.indexstatemanagement.util.isFailed
7374
import org.opensearch.indexmanagement.indexstatemanagement.util.isPolicyCompleted
7475
import org.opensearch.indexmanagement.indexstatemanagement.util.managedIndexConfigIndexRequest
7576
import org.opensearch.indexmanagement.indexstatemanagement.util.updateEnableManagedIndexRequest
76-
import org.opensearch.indexmanagement.migration.ISMTemplateService
7777
import org.opensearch.indexmanagement.opensearchapi.IndexManagementSecurityContext
7878
import org.opensearch.indexmanagement.opensearchapi.contentParser
7979
import org.opensearch.indexmanagement.opensearchapi.parseFromSearchResponse
@@ -128,17 +128,26 @@ class ManagedIndexCoordinator(
128128
private var scheduledTemplateMigration: Scheduler.Cancellable? = null
129129

130130
@Volatile private var lastFullSweepTimeNano = System.nanoTime()
131+
131132
@Volatile private var indexStateManagementEnabled = INDEX_STATE_MANAGEMENT_ENABLED.get(settings)
133+
132134
@Volatile private var metadataServiceEnabled = METADATA_SERVICE_ENABLED.get(settings)
135+
133136
@Volatile private var sweepPeriod = SWEEP_PERIOD.get(settings)
137+
134138
@Volatile private var retryPolicy =
135139
BackoffPolicy.constantBackoff(COORDINATOR_BACKOFF_MILLIS.get(settings), COORDINATOR_BACKOFF_COUNT.get(settings))
140+
136141
@Volatile private var templateMigrationEnabled: Boolean = true
142+
137143
@Volatile private var templateMigrationEnabledSetting = TEMPLATE_MIGRATION_CONTROL.get(settings)
144+
138145
@Volatile private var jobInterval = JOB_INTERVAL.get(settings)
146+
139147
@Volatile private var jobJitter = JITTER.get(settings)
140148

141149
@Volatile private var isMaster = false
150+
142151
@Volatile private var onMasterTimeStamp: Long = 0L
143152

144153
init {
@@ -168,8 +177,7 @@ class ManagedIndexCoordinator(
168177
if (!templateMigrationEnabled) scheduledTemplateMigration?.cancel()
169178
else initTemplateMigration(it)
170179
}
171-
clusterService.clusterSettings.addSettingsUpdateConsumer(COORDINATOR_BACKOFF_MILLIS, COORDINATOR_BACKOFF_COUNT) {
172-
millis, count ->
180+
clusterService.clusterSettings.addSettingsUpdateConsumer(COORDINATOR_BACKOFF_MILLIS, COORDINATOR_BACKOFF_COUNT) { millis, count ->
173181
retryPolicy = BackoffPolicy.constantBackoff(millis, count)
174182
}
175183
}
@@ -556,14 +564,15 @@ class ManagedIndexCoordinator(
556564

557565
logger.info("Performing ISM template migration.")
558566
if (enableSetting == 0L) {
559-
if (onMasterTimeStamp != 0L)
567+
if (onMasterTimeStamp != 0L) {
560568
templateService.doMigration(Instant.ofEpochMilli(onMasterTimeStamp))
561-
else {
569+
} else {
562570
logger.error("No valid onMaster time cached, cancel ISM template migration job.")
563571
scheduledTemplateMigration?.cancel()
564572
}
565-
} else
573+
} else {
566574
templateService.doMigration(Instant.ofEpochMilli(enableSetting))
575+
}
567576
} catch (e: Exception) {
568577
logger.error("Failed to migrate ISM template", e)
569578
}
@@ -596,7 +605,8 @@ class ManagedIndexCoordinator(
596605

597606
// Get the matching policyIds for applicable indices
598607
val updateMatchingIndicesReqs = createManagedIndexRequests(
599-
clusterService.state(), unManagedIndices.map { (indexName, _) -> indexName }
608+
clusterService.state(),
609+
unManagedIndices.map { (indexName, _) -> indexName }
600610
)
601611

602612
// check all managed indices, if the index has already been deleted
@@ -701,7 +711,10 @@ class ManagedIndexCoordinator(
701711
mRes.forEach {
702712
if (it.response.isExists) {
703713
result[it.id] = contentParser(it.response.sourceAsBytesRef).parseWithType(
704-
it.response.id, it.response.seqNo, it.response.primaryTerm, ManagedIndexConfig.Companion::parse
714+
it.response.id,
715+
it.response.seqNo,
716+
it.response.primaryTerm,
717+
ManagedIndexConfig.Companion::parse
705718
)
706719
}
707720
}

‎src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/ManagedIndexRunner.kt‎

Lines changed: 31 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -118,10 +118,13 @@ object ManagedIndexRunner :
118118
private lateinit var extensionStatusChecker: ExtensionStatusChecker
119119
private lateinit var indexMetadataProvider: IndexMetadataProvider
120120
private var indexStateManagementEnabled: Boolean = DEFAULT_ISM_ENABLED
121+
121122
@Suppress("MagicNumber")
122123
private val savePolicyRetryPolicy = BackoffPolicy.exponentialBackoff(TimeValue.timeValueMillis(250), 3)
124+
123125
@Suppress("MagicNumber")
124126
private val updateMetaDataRetryPolicy = BackoffPolicy.exponentialBackoff(TimeValue.timeValueMillis(250), 3)
127+
125128
@Suppress("MagicNumber")
126129
private val errorNotificationRetryPolicy = BackoffPolicy.exponentialBackoff(TimeValue.timeValueMillis(250), 3)
127130
private var jobInterval: Int = DEFAULT_JOB_INTERVAL
@@ -345,7 +348,8 @@ object ManagedIndexRunner :
345348
val info = mapOf("message" to "Previous action was not able to update IndexMetaData.")
346349
val updated = updateManagedIndexMetaData(
347350
managedIndexMetaData.copy(
348-
policyRetryInfo = PolicyRetryInfoMetaData(true, 0), info = info
351+
policyRetryInfo = PolicyRetryInfoMetaData(true, 0),
352+
info = info
349353
)
350354
)
351355
if (updated.metadataSaved) disableManagedIndexConfig(managedIndexConfig)
@@ -359,7 +363,8 @@ object ManagedIndexRunner :
359363
val info = mapOf("message" to "Failed to execute action=${action?.type} as extension [$actionExtensionName] is not enabled.")
360364
val updated = updateManagedIndexMetaData(
361365
managedIndexMetaData.copy(
362-
policyRetryInfo = PolicyRetryInfoMetaData(true, 0), info = info
366+
policyRetryInfo = PolicyRetryInfoMetaData(true, 0),
367+
info = info
363368
)
364369
)
365370
if (updated.metadataSaved) disableManagedIndexConfig(managedIndexConfig)
@@ -372,7 +377,8 @@ object ManagedIndexRunner :
372377
val info = mapOf("message" to "Attempted to execute action=${action?.type} which is not allowed.")
373378
val updated = updateManagedIndexMetaData(
374379
managedIndexMetaData.copy(
375-
policyRetryInfo = PolicyRetryInfoMetaData(true, 0), info = info
380+
policyRetryInfo = PolicyRetryInfoMetaData(true, 0),
381+
info = info
376382
)
377383
)
378384
if (updated.metadataSaved) disableManagedIndexConfig(managedIndexConfig)
@@ -388,7 +394,10 @@ object ManagedIndexRunner :
388394
// Step null check is done in getStartingManagedIndexMetaData
389395
withClosableContext(
390396
IndexManagementSecurityContext(
391-
managedIndexConfig.id, settings, threadPool.threadContext, managedIndexConfig.policy.user
397+
managedIndexConfig.id,
398+
settings,
399+
threadPool.threadContext,
400+
managedIndexConfig.policy.user
392401
)
393402
) {
394403
step.preExecute(logger, stepContext.getUpdatedContext(startingManagedIndexMetaData)).execute().postExecute(logger)
@@ -476,8 +485,10 @@ object ManagedIndexRunner :
476485
// Intellij complains about createParser/parseWithType blocking because it sees they throw IOExceptions
477486
return withContext(Dispatchers.IO) {
478487
val xcp = XContentHelper.createParser(
479-
xContentRegistry, LoggingDeprecationHandler.INSTANCE,
480-
policySource, XContentType.JSON
488+
xContentRegistry,
489+
LoggingDeprecationHandler.INSTANCE,
490+
policySource,
491+
XContentType.JSON
481492
)
482493
xcp.parseWithType(getResponse.id, getResponse.seqNo, getResponse.primaryTerm, Policy.Companion::parse)
483494
}
@@ -504,8 +515,11 @@ object ManagedIndexRunner :
504515
@Suppress("TooGenericExceptionCaught")
505516
private suspend fun savePolicyToManagedIndexConfig(managedIndexConfig: ManagedIndexConfig, policy: Policy): Boolean {
506517
val updatedManagedIndexConfig = managedIndexConfig.copy(
507-
policyID = policy.id, policy = policy,
508-
policySeqNo = policy.seqNo, policyPrimaryTerm = policy.primaryTerm, changePolicy = null
518+
policyID = policy.id,
519+
policy = policy,
520+
policySeqNo = policy.seqNo,
521+
policyPrimaryTerm = policy.primaryTerm,
522+
changePolicy = null
509523
)
510524
val indexRequest = managedIndexConfigIndexRequest(updatedManagedIndexConfig)
511525
var savedPolicy = false
@@ -605,8 +619,8 @@ object ManagedIndexRunner :
605619
// this is an edge case where a user deletes the job config or index and we already have a policySeqNo/primaryTerm
606620
// in the metadata, in this case we just want to say we successfully initialized the policy again but we will not
607621
// modify the state, action, etc. so it can resume where it left off
608-
managedIndexMetaData.policySeqNo == policy.seqNo && managedIndexMetaData.policyPrimaryTerm == policy.primaryTerm
609-
&& managedIndexMetaData.policyID == policy.id ->
622+
managedIndexMetaData.policySeqNo == policy.seqNo && managedIndexMetaData.policyPrimaryTerm == policy.primaryTerm &&
623+
managedIndexMetaData.policyID == policy.id ->
610624
// If existing PolicySeqNo and PolicyPrimaryTerm is equal to cached Policy then no issue.
611625
managedIndexMetaData.copy(
612626
policyRetryInfo = PolicyRetryInfoMetaData(failed = false, consumedRetries = 0),
@@ -688,7 +702,6 @@ object ManagedIndexRunner :
688702
managedIndexMetaData: ManagedIndexMetaData,
689703
actionToExecute: Action?
690704
) {
691-
692705
// should never happen since we only call this if there is a changePolicy, but we'll do it to make changePolicy non null
693706
val changePolicy = managedIndexConfig.changePolicy
694707
if (changePolicy == null) {
@@ -709,8 +722,13 @@ object ManagedIndexRunner :
709722
// if the action to execute is transition then set the actionMetaData to a new transition metadata to reflect we are
710723
// in transition (in case we triggered change policy from entering transition) or to reflect this is a new policy transition phase
711724
val newTransitionMetaData = ActionMetaData(
712-
TransitionsAction.name, Instant.now().toEpochMilli(), -1,
713-
false, 0, 0, null
725+
TransitionsAction.name,
726+
Instant.now().toEpochMilli(),
727+
-1,
728+
false,
729+
0,
730+
0,
731+
null
714732
)
715733
val actionMetaData = if (actionToExecute?.type == TransitionsAction.name) {
716734
newTransitionMetaData

0 commit comments

Comments
 (0)