Skip to content

Commit 121bd05

Browse files
masseykemasseyke
authored
Adding deprecation checks for geo_shape parameters (#76627)
The following properties have been removed from the geo_shape data type in 8.0: "strategy", "tree", "tree_levels", "precision", "distance_error_pct", "points_only". This PR adds a deprecation info API check for indexes and templates that have any of these properties in their mappings. Relates #42404 #70850
1 parent 0877bbc commit 121bd05

File tree

5 files changed

+279
-2
lines changed

5 files changed

+279
-2
lines changed

‎x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecks.java

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,25 +10,32 @@
1010
import org.apache.logging.log4j.LogManager;
1111
import org.apache.logging.log4j.Logger;
1212
import org.elasticsearch.cluster.ClusterState;
13+
import org.elasticsearch.cluster.metadata.ComponentTemplate;
14+
import org.elasticsearch.cluster.metadata.IndexTemplateMetadata;
1315
import org.elasticsearch.cluster.metadata.MappingMetadata;
1416
import org.elasticsearch.common.Strings;
1517
import org.elasticsearch.common.collect.ImmutableOpenMap;
1618
import org.elasticsearch.common.compress.CompressedXContent;
1719
import org.elasticsearch.common.xcontent.XContentHelper;
20+
import org.elasticsearch.common.xcontent.XContentType;
1821
import org.elasticsearch.core.TimeValue;
22+
import org.elasticsearch.core.Tuple;
1923
import org.elasticsearch.index.IndexSettings;
2024
import org.elasticsearch.index.mapper.FieldNamesFieldMapper;
25+
import org.elasticsearch.index.mapper.LegacyGeoShapeFieldMapper;
2126
import org.elasticsearch.ingest.IngestService;
2227
import org.elasticsearch.ingest.PipelineConfiguration;
2328

2429
import java.util.ArrayList;
30+
import java.util.Collections;
2531
import java.util.HashSet;
2632
import java.util.List;
2733
import java.util.Map;
2834
import java.util.Objects;
2935
import java.util.Set;
3036
import java.util.concurrent.atomic.AtomicInteger;
3137
import java.util.stream.Collectors;
38+
import java.util.stream.StreamSupport;
3239

3340
import static org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_INCLUDE_RELOCATIONS_SETTING;
3441
import static org.elasticsearch.search.SearchModule.INDICES_MAX_CLAUSE_COUNT_SETTING;
@@ -201,4 +208,86 @@ static DeprecationIssue checkClusterRoutingAllocationIncludeRelocationsSetting(f
201208
DeprecationIssue.Level.WARNING
202209
);
203210
}
211+
212+
@SuppressWarnings("unchecked")
213+
private static String getDetailsMessageForComponentTemplates(Map<String, ComponentTemplate> componentTemplates) {
214+
String detailsForComponentTemplates =
215+
componentTemplates.entrySet().stream().map((templateCursor) -> {
216+
String templateName = templateCursor.getKey();
217+
ComponentTemplate componentTemplate = templateCursor.getValue();
218+
CompressedXContent mappings = componentTemplate.template().mappings();
219+
if (mappings != null) {
220+
Tuple<XContentType, Map<String, Object>> tuple = XContentHelper.convertToMap(mappings.uncompressed(), true,
221+
XContentType.JSON);
222+
Map<String, Object> mappingAsMap = tuple.v2();
223+
List<String> messages = mappingAsMap == null ? Collections.emptyList() :
224+
IndexDeprecationChecks.findInPropertiesRecursively(LegacyGeoShapeFieldMapper.CONTENT_TYPE,
225+
mappingAsMap,
226+
IndexDeprecationChecks::isGeoShapeFieldWithDeprecatedParam,
227+
IndexDeprecationChecks::formatDeprecatedGeoShapeParamMessage);
228+
if (messages.isEmpty() == false) {
229+
String messageForMapping =
230+
"mappings in component template " + templateName + " contains deprecated geo_shape properties. " +
231+
messages.stream().collect(Collectors.joining("; "));
232+
return messageForMapping;
233+
}
234+
}
235+
return null;
236+
}).filter(messageForTemplate -> Strings.isEmpty(messageForTemplate) == false).collect(Collectors.joining("; "));
237+
return detailsForComponentTemplates;
238+
}
239+
240+
@SuppressWarnings("unchecked")
241+
private static String getDetailsMessageForIndexTemplates(ImmutableOpenMap<String, IndexTemplateMetadata> indexTemplates) {
242+
String detailsForIndexTemplates =
243+
StreamSupport.stream(indexTemplates.spliterator(), false).map((templateCursor) -> {
244+
String templateName = templateCursor.key;
245+
IndexTemplateMetadata indexTemplateMetadata = templateCursor.value;
246+
String messageForTemplate =
247+
StreamSupport.stream(indexTemplateMetadata.getMappings().spliterator(), false).map((mappingCursor) -> {
248+
CompressedXContent mapping = mappingCursor.value;
249+
Tuple<XContentType, Map<String, Object>> tuple = XContentHelper.convertToMap(mapping.uncompressed(), true,
250+
XContentType.JSON);
251+
Map<String, Object> mappingAsMap = (Map<String, Object>) tuple.v2().get("_doc");
252+
List<String> messages = mappingAsMap == null ? Collections.emptyList() :
253+
IndexDeprecationChecks.findInPropertiesRecursively(LegacyGeoShapeFieldMapper.CONTENT_TYPE,
254+
mappingAsMap,
255+
IndexDeprecationChecks::isGeoShapeFieldWithDeprecatedParam,
256+
IndexDeprecationChecks::formatDeprecatedGeoShapeParamMessage);
257+
return messages;
258+
}).filter(messages -> messages.isEmpty() == false).map(messages -> {
259+
String messageForMapping =
260+
"mappings in index template " + templateName + " contains deprecated geo_shape properties. " +
261+
messages.stream().collect(Collectors.joining("; "));
262+
return messageForMapping;
263+
}).collect(Collectors.joining("; "));
264+
return messageForTemplate;
265+
}).filter(messageForTemplate -> Strings.isEmpty(messageForTemplate) == false).collect(Collectors.joining("; "));
266+
return detailsForIndexTemplates;
267+
}
268+
269+
@SuppressWarnings("unchecked")
270+
static DeprecationIssue checkGeoShapeTemplates(final ClusterState clusterState) {
271+
String detailsForComponentTemplates = getDetailsMessageForComponentTemplates(clusterState.getMetadata().componentTemplates());
272+
String detailsForIndexTemplates = getDetailsMessageForIndexTemplates(clusterState.getMetadata().getTemplates());
273+
boolean deprecationInComponentTemplates = Strings.isEmpty(detailsForComponentTemplates) == false;
274+
boolean deprecationInIndexTemplates = Strings.isEmpty(detailsForIndexTemplates) == false;
275+
String url = "https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-8.0.html#breaking_80_mappings_changes";
276+
if (deprecationInComponentTemplates && deprecationInIndexTemplates) {
277+
String message = "component templates and index templates contain deprecated geo_shape properties that must be removed";
278+
String details = detailsForComponentTemplates + "; " + detailsForIndexTemplates;
279+
return new DeprecationIssue(DeprecationIssue.Level.CRITICAL, message, url, details, false,
280+
null);
281+
} if (deprecationInComponentTemplates == false && deprecationInIndexTemplates) {
282+
String message = "index templates contain deprecated geo_shape properties that must be removed";
283+
return new DeprecationIssue(DeprecationIssue.Level.CRITICAL, message, url, detailsForIndexTemplates, false,
284+
null);
285+
} else if (deprecationInIndexTemplates == false && deprecationInComponentTemplates) {
286+
String message = "component templates contain deprecated geo_shape properties that must be removed";
287+
return new DeprecationIssue(DeprecationIssue.Level.CRITICAL, message, url, detailsForComponentTemplates, false,
288+
null);
289+
} else {
290+
return null;
291+
}
292+
}
204293
}

‎x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,8 @@ private DeprecationChecks() {
3838
ClusterDeprecationChecks::checkPollIntervalTooLow,
3939
ClusterDeprecationChecks::checkTemplatesWithFieldNamesDisabled,
4040
ClusterDeprecationChecks::checkTemplatesWithMultipleTypes,
41-
ClusterDeprecationChecks::checkClusterRoutingAllocationIncludeRelocationsSetting
41+
ClusterDeprecationChecks::checkClusterRoutingAllocationIncludeRelocationsSetting,
42+
ClusterDeprecationChecks::checkGeoShapeTemplates
4243
));
4344

4445
static final List<NodeDeprecationCheck<Settings, PluginsAndModules, ClusterState, XPackLicenseState, DeprecationIssue>>
@@ -111,7 +112,8 @@ private DeprecationChecks() {
111112
IndexDeprecationChecks::checkIndexDataPath,
112113
IndexDeprecationChecks::indexingSlowLogLevelSettingCheck,
113114
IndexDeprecationChecks::searchSlowLogLevelSettingCheck,
114-
IndexDeprecationChecks::storeTypeSettingCheck
115+
IndexDeprecationChecks::storeTypeSettingCheck,
116+
IndexDeprecationChecks::checkGeoShapeMappings
115117
));
116118

117119
/**

‎x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecks.java

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import org.elasticsearch.index.IndexingSlowLog;
2020
import org.elasticsearch.index.SearchSlowLog;
2121
import org.elasticsearch.index.SlowLogLevel;
22+
import org.elasticsearch.index.mapper.LegacyGeoShapeFieldMapper;
2223

2324
import java.util.ArrayList;
2425
import java.util.Collections;
@@ -31,6 +32,7 @@
3132
import java.util.function.BiConsumer;
3233
import java.util.function.BiFunction;
3334
import java.util.function.Function;
35+
import java.util.stream.Collectors;
3436

3537

3638
/**
@@ -317,4 +319,42 @@ static DeprecationIssue storeTypeSettingCheck(IndexMetadata indexMetadata) {
317319
}
318320
return null;
319321
}
322+
323+
protected static boolean isGeoShapeFieldWithDeprecatedParam(Map<?, ?> property) {
324+
return LegacyGeoShapeFieldMapper.CONTENT_TYPE.equals(property.get("type")) &&
325+
LegacyGeoShapeFieldMapper.DEPRECATED_PARAMETERS.stream().anyMatch(deprecatedParameter ->
326+
property.containsKey(deprecatedParameter)
327+
);
328+
}
329+
330+
protected static String formatDeprecatedGeoShapeParamMessage(String type, Map.Entry<?, ?> entry) {
331+
String fieldName = entry.getKey().toString();
332+
Map<?, ?> value = (Map<?, ?>) entry.getValue();
333+
return LegacyGeoShapeFieldMapper.DEPRECATED_PARAMETERS.stream()
334+
.filter(deprecatedParameter -> value.containsKey(deprecatedParameter))
335+
.map(deprecatedParameter -> String.format(Locale.ROOT, "parameter [%s] in field [%s]", type, deprecatedParameter, fieldName))
336+
.collect(Collectors.joining("; "));
337+
}
338+
339+
@SuppressWarnings("unchecked")
340+
static DeprecationIssue checkGeoShapeMappings(IndexMetadata indexMetadata) {
341+
if (indexMetadata == null || indexMetadata.mapping() == null) {
342+
return null;
343+
}
344+
Map<String, Object> sourceAsMap = indexMetadata.mapping().getSourceAsMap();
345+
List<String> messages = findInPropertiesRecursively(LegacyGeoShapeFieldMapper.CONTENT_TYPE, sourceAsMap,
346+
IndexDeprecationChecks::isGeoShapeFieldWithDeprecatedParam,
347+
IndexDeprecationChecks::formatDeprecatedGeoShapeParamMessage);
348+
if (messages.isEmpty()) {
349+
return null;
350+
} else {
351+
String message = String.format(Locale.ROOT,"mappings for index %s contains deprecated geo_shape properties that must be " +
352+
"removed", indexMetadata.getIndex().getName());
353+
String details = String.format(Locale.ROOT,
354+
"The following geo_shape parameters must be removed from %s: [%s]", indexMetadata.getIndex().getName(),
355+
messages.stream().collect(Collectors.joining("; ")));
356+
String url = "https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-8.0.html#breaking_80_mappings_changes";
357+
return new DeprecationIssue(DeprecationIssue.Level.CRITICAL, message, url, details, false, null);
358+
}
359+
}
320360
}

‎x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecksTests.java

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,15 @@
99
import org.elasticsearch.action.ingest.PutPipelineRequest;
1010
import org.elasticsearch.cluster.ClusterName;
1111
import org.elasticsearch.cluster.ClusterState;
12+
import org.elasticsearch.cluster.metadata.AliasMetadata;
13+
import org.elasticsearch.cluster.metadata.ComponentTemplate;
1214
import org.elasticsearch.cluster.metadata.IndexTemplateMetadata;
1315
import org.elasticsearch.cluster.metadata.Metadata;
16+
import org.elasticsearch.cluster.metadata.Template;
1417
import org.elasticsearch.common.Strings;
1518
import org.elasticsearch.common.bytes.BytesArray;
1619
import org.elasticsearch.common.collect.ImmutableOpenMap;
20+
import org.elasticsearch.common.compress.CompressedXContent;
1721
import org.elasticsearch.common.settings.Settings;
1822
import org.elasticsearch.common.xcontent.XContentBuilder;
1923
import org.elasticsearch.common.xcontent.XContentType;
@@ -24,6 +28,7 @@
2428

2529
import java.io.IOException;
2630
import java.util.Collections;
31+
import java.util.HashMap;
2732
import java.util.List;
2833
import java.util.Locale;
2934

@@ -355,4 +360,83 @@ public void testClusterRoutingAllocationIncludeRelocationsSetting() {
355360

356361
assertWarnings(expectedWarning);
357362
}
363+
364+
public void testCheckGeoShapeMappings() throws Exception {
365+
// First, testing only an index template:
366+
IndexTemplateMetadata indexTemplateMetadata = IndexTemplateMetadata.builder("single-type")
367+
.patterns(Collections.singletonList("foo"))
368+
.putMapping("_doc", "{\n" +
369+
" \"_doc\":{\n" +
370+
" \"properties\":{\n" +
371+
" \"nested_field\":{\n" +
372+
" \"type\":\"nested\",\n" +
373+
" \"properties\":{\n" +
374+
" \"location\":{\n" +
375+
" \"type\":\"geo_shape\",\n" +
376+
" \"strategy\":\"recursive\",\n" +
377+
" \"points_only\":true\n" +
378+
" }\n" +
379+
" }\n" +
380+
" }\n" +
381+
" }\n" +
382+
" }\n" +
383+
"}")
384+
.build();
385+
ImmutableOpenMap<String, IndexTemplateMetadata> templates = ImmutableOpenMap.<String, IndexTemplateMetadata>builder()
386+
.fPut("single-type", indexTemplateMetadata)
387+
.build();
388+
Metadata badMetadata = Metadata.builder()
389+
.templates(templates)
390+
.build();
391+
ClusterState badState = ClusterState.builder(new ClusterName("test")).metadata(badMetadata).build();
392+
DeprecationIssue issue = ClusterDeprecationChecks.checkGeoShapeTemplates(badState);
393+
394+
assertThat(issue, equalTo(
395+
new DeprecationIssue(DeprecationIssue.Level.CRITICAL,
396+
"index templates contain deprecated geo_shape properties that must be removed",
397+
"https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-8.0.html#breaking_80_mappings_changes",
398+
"mappings in index template single-type contains deprecated geo_shape properties. [parameter [geo_shape] in field " +
399+
"[points_only]; parameter [geo_shape] in field [strategy]]", false, null)
400+
));
401+
402+
// Second, testing only a component template:
403+
String templateName = "my-template";
404+
Settings settings = Settings.builder().put("index.number_of_shards", 1).build();
405+
CompressedXContent mappings = new CompressedXContent("{\"properties\":{\"location\":{\"type\":\"geo_shape\", " +
406+
"\"strategy\":\"recursive\", \"points_only\":true}}}");
407+
AliasMetadata alias = AliasMetadata.builder("alias").writeIndex(true).build();
408+
Template template = new Template(settings, mappings, Collections.singletonMap("alias", alias));
409+
ComponentTemplate componentTemplate = new ComponentTemplate(template, 1L, new HashMap<>());
410+
badMetadata = Metadata.builder()
411+
.componentTemplates(Collections.singletonMap(templateName, componentTemplate))
412+
.build();
413+
badState = ClusterState.builder(new ClusterName("test")).metadata(badMetadata).build();
414+
issue = ClusterDeprecationChecks.checkGeoShapeTemplates(badState);
415+
416+
assertThat(issue, equalTo(
417+
new DeprecationIssue(DeprecationIssue.Level.CRITICAL,
418+
"component templates contain deprecated geo_shape properties that must be removed",
419+
"https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-8.0.html#breaking_80_mappings_changes",
420+
"mappings in component template my-template contains deprecated geo_shape properties. [parameter [geo_shape] in field " +
421+
"[points_only]; parameter [geo_shape] in field [strategy]]", false, null)
422+
));
423+
424+
// Third, trying a component template and an index template:
425+
badMetadata = Metadata.builder()
426+
.componentTemplates(Collections.singletonMap(templateName, componentTemplate))
427+
.templates(templates)
428+
.build();
429+
badState = ClusterState.builder(new ClusterName("test")).metadata(badMetadata).build();
430+
issue = ClusterDeprecationChecks.checkGeoShapeTemplates(badState);
431+
432+
assertThat(issue, equalTo(
433+
new DeprecationIssue(DeprecationIssue.Level.CRITICAL,
434+
"component templates and index templates contain deprecated geo_shape properties that must be removed",
435+
"https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-8.0.html#breaking_80_mappings_changes",
436+
"mappings in component template my-template contains deprecated geo_shape properties. [parameter [geo_shape] in field " +
437+
"[points_only]; parameter [geo_shape] in field [strategy]]; mappings in index template single-type contains " +
438+
"deprecated geo_shape properties. [parameter [geo_shape] in field [points_only]; parameter [geo_shape] in field " +
439+
"[strategy]]", false, null)
440+
));
441+
}
358442
}

0 commit comments

Comments
 (0)