Error On elastic roll up job :Failed to validate the source index mappings #69
Description
Tuesday Jun 01, 2021 at 17:26 GMT
Originally opened as opendistro-for-elasticsearch/index-management#449
team ,
I have created roll up policy successful and while running roll up job i am getting following error from the code ..
"Failed to validate the source index mappings."
Upon debugging from the source and printing the exception ..looks like we are assuming _doc in the mappings, hope _doc was removed in the latest elastic versions .Please clarify if I am missing or let us any known defect for this.
here is the source which got modified and the exception which we enabled to print in the logs
roll up policy which we tried
curl -XPUT "localhost:9200/_opendistro/_rollup/jobs/latest_stats_roll_up" -H 'Content-Type: application/json' -d'{"rollup":{"enabled":true,"schedule":{"interval":{"period":1,"unit":"Minutes"}},"description":"An example policy that rolls up the sample ecommerce data","source_index":"fmstats_2021-03-29","target_index":"latest_stats_rollup","page_size":1000,"delay":0,"continuous":false,"dimensions":[{"date_histogram":{"source_field":"timestamp","fixed_interval":"60m"}},{"terms":{"source_field":"portIdToClusterId"}},{"terms":{"source_field":"alias"}}],"metrics":[{"source_field":"port.rx.packets","metrics":[{"max":{}}]},{"source_field":"port.tx.packets","metrics":[{"max":{}}]}]}}'
--- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/rollup/RollupMapperService.kt
+++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/rollup/RollupMapperService.kt
@@ -45,6 +45,8 @@ import org.elasticsearch.cluster.service.ClusterService
import org.elasticsearch.common.settings.Settings
import org.elasticsearch.common.xcontent.XContentType
import org.elasticsearch.transport.RemoteTransportException
+import java.io.PrintWriter
+import java.io.StringWriter
// TODO: Validation of fields across source and target indices overwriting existing rollup data
// and type validation using mappings from source index
@@ -146,7 +148,7 @@ class RollupMapperService(
return RollupJobValidationResult.Failure(getMappingsResult.message, getMappingsResult.cause)
}
**val indexMapping: MappingMetadata = res.mappings[index][_DOC]**
val indexMappingSource = indexMapping.sourceAsMap
val issues = mutableSetOf<String>()
@@ -183,7 +185,12 @@ class RollupMapperService(
RollupJobValidationResult.Invalid("Invalid mappings for index [$index] because $issues")
}
} catch (e: Exception) {
- return RollupJobValidationResult.Failure("Failed to validate the source index mappings", e)
+ val errorMessage = "Failed to validate the source index mappings"
+ val sw = StringWriter()
+ e.printStackTrace(PrintWriter(sw))
+ val exceptionAsString = sw.toString()
+ logger.error(errorMessage, e)
+ return RollupJobValidationResult.Failure(exceptionAsString, e)
}
}
java.lang.IllegalStateException: res.mappings[index][_DOC] must not be null
at com.amazon.opendistroforelasticsearch.indexmanagement.rollup.RollupMapperService.isSourceIndexMappingsValid(RollupMapperService.kt:151) [opendistro-index-management-1.13.2.0-SNAPSHOT.jar:1.13.2.0-SNAPSHOT]
at com.amazon.opendistroforelasticsearch.indexmanagement.rollup.RollupMapperService.isSourceIndexValid(RollupMapperService.kt:132) [opendistro-index-management-1.13.2.0-SNAPSHOT.jar:1.13.2.0-SNAPSHOT]
at com.amazon.opendistroforelasticsearch.indexmanagement.rollup.RollupRunner.isJobValid(RollupRunner.kt:401) [opendistro-index-management-1.13.2.0-SNAPSHOT.jar:1.13.2.0-SNAPSHOT]
at com.amazon.opendistroforelasticsearch.indexmanagement.rollup.RollupRunner.runRollupJob(RollupRunner.kt:223) [opendistro-index-management-1.13.2.0-SNAPSHOT.jar:1.13.2.0-SNAPSHOT]
at com.amazon.opendistroforelasticsearch.indexmanagement.rollup.RollupRunner$runJob$1.invokeSuspend(RollupRunner.kt:160) [opendistro-index-management-1.13.2.0-SNAPSHOT.jar:1.13.2.0-SNAPSHOT]
at kotlin.coroutines.jvm.internal.BaseContinuationImpl.resumeWith(ContinuationImpl.kt:33) [kotlin-stdlib-1.3.72.jar:1.3.72-release-468 (1.3.72)]
at kotlinx.coroutines.DispatchedTask.run(DispatchedTask.kt:56) [kotlinx-coroutines-core-1.3.7.jar:?]
at kotlinx.coroutines.scheduling.CoroutineScheduler.runSafely(CoroutineScheduler.kt:571) [kotlinx-coroutines-core-1.3.7.jar:?]
at kotlinx.coroutines.scheduling.CoroutineScheduler$Worker.executeTask(CoroutineScheduler.kt:738) [kotlinx-coroutines-core-1.3.7.jar:?]
at kotlinx.coroutines.scheduling.CoroutineScheduler$Worker.runWorker(CoroutineScheduler.kt:678) [kotlinx-coroutines-core-1.3.7.jar:?]
at kotlinx.coroutines.scheduling.CoroutineScheduler$Worker.run(CoroutineScheduler.kt:665) [kotlinx-coroutines-core-1.3.7.jar:?]
[2021-06-01T16:47:13,795][ERROR][c.a.o.i.r.RollupRunner ] [fmha1] Failed to validate [latest_stats_roll_up]: [java.lang.IllegalStateException: res.mappings[index][_DOC] must not be null
at com.amazon.opendistroforelasticsearch.indexmanagement.rollup.RollupMapperService.isSourceIndexMappingsValid(RollupMapperService.kt:151)
at com.amazon.opendistroforelasticsearch.indexmanagement.rollup.RollupMapperService.isSourceIndexValid(RollupMapperService.kt:132)
at com.amazon.opendistroforelasticsearch.indexmanagement.rollup.RollupRunner.isJobValid(RollupRunner.kt:401)
at com.amazon.opendistroforelasticsearch.indexmanagement.rollup.RollupRunner.runRollupJob(RollupRunner.kt:223)