Skip to content

Commit ffe61fb

Browse files
javannajavanna
authored
Move runtime fields stats to server (#69487)
Runtime fields usage is currently reported as part of the xpack feature usage API. Now that runtime fields are part of server, their corresponding stats can be moved to be part of the ordinary mapping stats exposed by the cluster stats API.
1 parent 172ca20 commit ffe61fb

File tree

22 files changed

+581
-803
lines changed

22 files changed

+581
-803
lines changed

‎docs/reference/cluster/stats.asciidoc

Lines changed: 69 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -492,6 +492,72 @@ Number of fields mapped to the field data type in selected nodes.
492492
`index_count`::
493493
(integer)
494494
Number of indices containing a mapping of the field data type in selected nodes.
495+
======
496+
497+
`runtime_field_types`::
498+
(array of objects)
499+
Contains statistics about <<runtime-mapping-fields, runtime field data types>> used in selected
500+
nodes.
501+
+
502+
.Properties of `runtime_field_types` objects
503+
[%collapsible%open]
504+
======
505+
`name`::
506+
(string)
507+
Field data type used in selected nodes.
508+
509+
`count`::
510+
(integer)
511+
Number of runtime fields mapped to the field data type in selected nodes.
512+
513+
`index_count`::
514+
(integer)
515+
Number of indices containing a mapping of the runtime field data type in selected nodes.
516+
517+
`scriptless_count`::
518+
(integer)
519+
Number of runtime fields that don't declare a script.
520+
521+
`shadowed_count`::
522+
(integer)
523+
Number of runtime fields that shadow an indexed field.
524+
525+
`lang`::
526+
(array of strings)
527+
Script languages used for the runtime fields scripts
528+
529+
`lines_max`::
530+
(integer)
531+
Maximum number of lines for a single runtime field script
532+
533+
`lines_total`::
534+
(integer)
535+
Total number of lines for the scripts that define the current runtime field data type
536+
537+
`chars_max`::
538+
(integer)
539+
Maximum number of characters for a single runtime field script
540+
541+
`chars_total`::
542+
(integer)
543+
Total number of characters for the scripts that define the current runtime field data type
544+
545+
`source_max`::
546+
(integer)
547+
Maximum number of accesses to _source for a single runtime field script
548+
549+
`source_total`::
550+
(integer)
551+
Total number of accesses to _source for the scripts that define the current runtime field data type
552+
553+
`doc_max`::
554+
(integer)
555+
Maximum number of accesses to doc_values for a single runtime field script
556+
557+
`doc_total`::
558+
(integer)
559+
Total number of accesses to doc_values for the scripts that define the current runtime field data type
560+
495561
======
496562
=====
497563
@@ -1220,7 +1286,8 @@ The API returns the following response:
12201286
"file_sizes": {}
12211287
},
12221288
"mappings": {
1223-
"field_types": []
1289+
"field_types": [],
1290+
"runtime_field_types": []
12241291
},
12251292
"analysis": {
12261293
"char_filter_types": [],
@@ -1363,6 +1430,7 @@ The API returns the following response:
13631430
// TESTRESPONSE[s/"count": \{[^\}]*\}/"count": $body.$_path/]
13641431
// TESTRESPONSE[s/"packaging_types": \[[^\]]*\]/"packaging_types": $body.$_path/]
13651432
// TESTRESPONSE[s/"field_types": \[[^\]]*\]/"field_types": $body.$_path/]
1433+
// TESTRESPONSE[s/"runtime_field_types": \[[^\]]*\]/"runtime_field_types": $body.$_path/]
13661434
// TESTRESPONSE[s/: true|false/: $body.$_path/]
13671435
// TESTRESPONSE[s/: (\-)?[0-9]+/: $body.$_path/]
13681436
// TESTRESPONSE[s/: "[^"]*"/: $body.$_path/]

‎docs/reference/rest-api/info.asciidoc

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -114,10 +114,6 @@ Example response:
114114
"available": true,
115115
"enabled": true
116116
},
117-
"runtime_fields": {
118-
"available": true,
119-
"enabled": true
120-
},
121117
"searchable_snapshots" : {
122118
"available" : true,
123119
"enabled" : true

‎docs/reference/rest-api/usage.asciidoc

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -392,11 +392,6 @@ GET /_xpack/usage
392392
"aggregate_metric" : {
393393
"available" : true,
394394
"enabled" : true
395-
},
396-
"runtime_fields" : {
397-
"available" : true,
398-
"enabled" : true,
399-
"field_types" : []
400395
}
401396
}
402397
------------------------------------------------------------

‎rest-api-spec/src/main/resources/rest-api-spec/test/cluster.stats/10_basic.yml

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,3 +83,86 @@
8383
cluster.stats: {}
8484

8585
- is_true: nodes.packaging_types
86+
---
87+
"get cluster stats without runtime fields":
88+
- skip:
89+
version: " - 7.99.99"
90+
reason: "cluster stats includes runtime fields from 8.0 on"
91+
- do:
92+
indices.create:
93+
index: sensor
94+
95+
- do: {cluster.stats: {}}
96+
- length: { indices.mappings.field_types: 0 }
97+
- length: { indices.mappings.runtime_field_types: 0 }
98+
99+
---
100+
"Usage stats with script-less runtime fields":
101+
- skip:
102+
version: " - 7.99.99"
103+
reason: "cluster stats includes runtime fields from 8.0 on"
104+
- do:
105+
indices.create:
106+
index: sensor
107+
body:
108+
mappings:
109+
runtime:
110+
message_from_source:
111+
type: keyword
112+
bad_map:
113+
type: double # shadows the bad_map field in properties
114+
message.text:
115+
type: keyword # shadows the message.text subfield in properties
116+
properties:
117+
message:
118+
type: keyword
119+
fields:
120+
text:
121+
type: text
122+
bad_map:
123+
type: long
124+
125+
- do: {cluster.stats: {}}
126+
- length: { indices.mappings.field_types: 3 }
127+
128+
- match: { indices.mappings.field_types.0.name: keyword }
129+
- match: { indices.mappings.field_types.0.count: 1 }
130+
- match: { indices.mappings.field_types.0.index_count: 1 }
131+
- match: { indices.mappings.field_types.1.name: long }
132+
- match: { indices.mappings.field_types.1.count: 1 }
133+
- match: { indices.mappings.field_types.1.index_count: 1 }
134+
- match: { indices.mappings.field_types.2.name: text }
135+
- match: { indices.mappings.field_types.2.count: 1 }
136+
- match: { indices.mappings.field_types.2.index_count: 1 }
137+
138+
139+
- length: { indices.mappings.runtime_field_types: 2 }
140+
141+
- match: { indices.mappings.runtime_field_types.0.name: double }
142+
- match: { indices.mappings.runtime_field_types.0.count: 1 }
143+
- match: { indices.mappings.runtime_field_types.0.index_count: 1 }
144+
- match: { indices.mappings.runtime_field_types.0.scriptless_count: 1 }
145+
- match: { indices.mappings.runtime_field_types.0.shadowed_count: 1 }
146+
- match: { indices.mappings.runtime_field_types.0.source_max: 0 }
147+
- match: { indices.mappings.runtime_field_types.0.source_total: 0 }
148+
- match: { indices.mappings.runtime_field_types.0.lines_max: 0 }
149+
- match: { indices.mappings.runtime_field_types.0.lines_total: 0 }
150+
- match: { indices.mappings.runtime_field_types.0.chars_max: 0 }
151+
- match: { indices.mappings.runtime_field_types.0.chars_total: 0 }
152+
- match: { indices.mappings.runtime_field_types.0.doc_max: 0 }
153+
- match: { indices.mappings.runtime_field_types.0.doc_total: 0 }
154+
155+
- match: { indices.mappings.runtime_field_types.1.name: keyword }
156+
- match: { indices.mappings.runtime_field_types.1.count: 2 }
157+
- match: { indices.mappings.runtime_field_types.1.index_count: 1 }
158+
- match: { indices.mappings.runtime_field_types.1.scriptless_count: 2 }
159+
- match: { indices.mappings.runtime_field_types.1.shadowed_count: 1 }
160+
- match: { indices.mappings.runtime_field_types.1.source_max: 0 }
161+
- match: { indices.mappings.runtime_field_types.1.source_total: 0 }
162+
- match: { indices.mappings.runtime_field_types.1.lines_max: 0 }
163+
- match: { indices.mappings.runtime_field_types.1.lines_total: 0 }
164+
- match: { indices.mappings.runtime_field_types.1.chars_max: 0 }
165+
- match: { indices.mappings.runtime_field_types.1.chars_total: 0 }
166+
- match: { indices.mappings.runtime_field_types.1.doc_max: 0 }
167+
- match: { indices.mappings.runtime_field_types.1.doc_total: 0 }
168+

‎server/src/main/java/org/elasticsearch/action/admin/cluster/stats/MappingStats.java

Lines changed: 82 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88

99
package org.elasticsearch.action.admin.cluster.stats;
1010

11+
import org.elasticsearch.Version;
1112
import org.elasticsearch.cluster.metadata.IndexMetadata;
1213
import org.elasticsearch.cluster.metadata.MappingMetadata;
1314
import org.elasticsearch.cluster.metadata.Metadata;
@@ -28,7 +29,10 @@
2829
import java.util.LinkedHashSet;
2930
import java.util.List;
3031
import java.util.Map;
32+
import java.util.Objects;
3133
import java.util.Set;
34+
import java.util.regex.Matcher;
35+
import java.util.regex.Pattern;
3236

3337
/**
3438
* Usage statistics about mappings usage.
@@ -40,6 +44,8 @@ public final class MappingStats implements ToXContentFragment, Writeable {
4044
*/
4145
public static MappingStats of(Metadata metadata, Runnable ensureNotCancelled) {
4246
Map<String, IndexFeatureStats> fieldTypes = new HashMap<>();
47+
Set<String> concreteFieldNames = new HashSet<>();
48+
Map<String, RuntimeFieldStats> runtimeFieldTypes = new HashMap<>();
4349
for (IndexMetadata indexMetadata : metadata) {
4450
ensureNotCancelled.run();
4551
if (indexMetadata.isSystem()) {
@@ -48,9 +54,11 @@ public static MappingStats of(Metadata metadata, Runnable ensureNotCancelled) {
4854
continue;
4955
}
5056
Set<String> indexFieldTypes = new HashSet<>();
57+
Set<String> indexRuntimeFieldTypes = new HashSet<>();
5158
MappingMetadata mappingMetadata = indexMetadata.mapping();
5259
if (mappingMetadata != null) {
5360
MappingVisitor.visitMapping(mappingMetadata.getSourceAsMap(), (field, fieldMapping) -> {
61+
concreteFieldNames.add(field);
5462
String type = null;
5563
Object typeO = fieldMapping.get("type");
5664
if (typeO != null) {
@@ -67,26 +75,83 @@ public static MappingStats of(Metadata metadata, Runnable ensureNotCancelled) {
6775
}
6876
}
6977
});
78+
79+
MappingVisitor.visitRuntimeMapping(mappingMetadata.getSourceAsMap(), (field, fieldMapping) -> {
80+
Object typeObject = fieldMapping.get("type");
81+
if (typeObject == null) {
82+
return;
83+
}
84+
String type = typeObject.toString();
85+
RuntimeFieldStats stats = runtimeFieldTypes.computeIfAbsent(type, RuntimeFieldStats::new);
86+
stats.count++;
87+
if (indexRuntimeFieldTypes.add(type)) {
88+
stats.indexCount++;
89+
}
90+
if (concreteFieldNames.contains(field)) {
91+
stats.shadowedCount++;
92+
}
93+
Object scriptObject = fieldMapping.get("script");
94+
if (scriptObject == null) {
95+
stats.scriptLessCount++;
96+
} else if (scriptObject instanceof Map) {
97+
Map<?, ?> script = (Map<?, ?>) scriptObject;
98+
Object sourceObject = script.get("source");
99+
if (sourceObject != null) {
100+
String scriptSource = sourceObject.toString();
101+
int chars = scriptSource.length();
102+
long lines = scriptSource.lines().count();
103+
int docUsages = countOccurrences(scriptSource, "doc[\\[\\.]");
104+
int sourceUsages = countOccurrences(scriptSource, "params\\._source");
105+
stats.update(chars, lines, sourceUsages, docUsages);
106+
}
107+
Object langObject = script.get("lang");
108+
if (langObject != null) {
109+
stats.scriptLangs.add(langObject.toString());
110+
}
111+
}
112+
});
70113
}
71114
}
72-
return new MappingStats(fieldTypes.values());
115+
return new MappingStats(fieldTypes.values(), runtimeFieldTypes.values());
116+
}
117+
118+
private static int countOccurrences(String script, String keyword) {
119+
int occurrences = 0;
120+
Pattern pattern = Pattern.compile(keyword);
121+
Matcher matcher = pattern.matcher(script);
122+
while (matcher.find()) {
123+
occurrences++;
124+
}
125+
return occurrences;
73126
}
74127

75128
private final Set<IndexFeatureStats> fieldTypeStats;
129+
private final Set<RuntimeFieldStats> runtimeFieldTypeStats;
76130

77-
MappingStats(Collection<IndexFeatureStats> fieldTypeStats) {
131+
MappingStats(Collection<IndexFeatureStats> fieldTypeStats, Collection<RuntimeFieldStats> runtimeFieldTypeStats) {
78132
List<IndexFeatureStats> stats = new ArrayList<>(fieldTypeStats);
79133
stats.sort(Comparator.comparing(IndexFeatureStats::getName));
80134
this.fieldTypeStats = Collections.unmodifiableSet(new LinkedHashSet<IndexFeatureStats>(stats));
135+
List<RuntimeFieldStats> runtimeStats = new ArrayList<>(runtimeFieldTypeStats);
136+
runtimeStats.sort(Comparator.comparing(RuntimeFieldStats::type));
137+
this.runtimeFieldTypeStats = Collections.unmodifiableSet(new LinkedHashSet<>(runtimeStats));
81138
}
82139

83140
MappingStats(StreamInput in) throws IOException {
84141
fieldTypeStats = Collections.unmodifiableSet(new LinkedHashSet<>(in.readList(IndexFeatureStats::new)));
142+
if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
143+
runtimeFieldTypeStats = Collections.unmodifiableSet(new LinkedHashSet<>(in.readList(RuntimeFieldStats::new)));
144+
} else {
145+
runtimeFieldTypeStats = Collections.emptySet();
146+
}
85147
}
86148

87149
@Override
88150
public void writeTo(StreamOutput out) throws IOException {
89151
out.writeCollection(fieldTypeStats);
152+
if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
153+
out.writeCollection(runtimeFieldTypeStats);
154+
}
90155
}
91156

92157
/**
@@ -96,6 +161,13 @@ public Set<IndexFeatureStats> getFieldTypeStats() {
96161
return fieldTypeStats;
97162
}
98163

164+
/**
165+
* Return stats about runtime field types.
166+
*/
167+
public Set<RuntimeFieldStats> getRuntimeFieldTypeStats() {
168+
return runtimeFieldTypeStats;
169+
}
170+
99171
@Override
100172
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
101173
builder.startObject("mappings");
@@ -104,6 +176,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
104176
st.toXContent(builder, params);
105177
}
106178
builder.endArray();
179+
builder.startArray("runtime_field_types");
180+
for (RuntimeFieldStats st : runtimeFieldTypeStats) {
181+
st.toXContent(builder, params);
182+
}
183+
builder.endArray();
107184
builder.endObject();
108185
return builder;
109186
}
@@ -119,11 +196,12 @@ public boolean equals(Object o) {
119196
return false;
120197
}
121198
MappingStats that = (MappingStats) o;
122-
return fieldTypeStats.equals(that.fieldTypeStats);
199+
return fieldTypeStats.equals(that.fieldTypeStats) &&
200+
runtimeFieldTypeStats.equals(that.runtimeFieldTypeStats);
123201
}
124202

125203
@Override
126204
public int hashCode() {
127-
return fieldTypeStats.hashCode();
205+
return Objects.hash(fieldTypeStats, runtimeFieldTypeStats);
128206
}
129207
}

‎server/src/main/java/org/elasticsearch/action/admin/cluster/stats/MappingVisitor.java

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,4 +52,21 @@ private static void visitMapping(Map<String, ?> mapping, String path, BiConsumer
5252
}
5353
}
5454

55+
public static void visitRuntimeMapping(Map<String, ?> mapping, BiConsumer<String, Map<String, ?>> runtimeFieldMappingConsumer) {
56+
Object runtimeObject = mapping.get("runtime");
57+
if (runtimeObject instanceof Map == false) {
58+
return;
59+
}
60+
@SuppressWarnings("unchecked")
61+
Map<String, ?> runtimeMappings = (Map<String, ?>) runtimeObject;
62+
for (String runtimeFieldName : runtimeMappings.keySet()) {
63+
Object runtimeFieldMappingObject = runtimeMappings.get(runtimeFieldName);
64+
if (runtimeFieldMappingObject instanceof Map == false) {
65+
continue;
66+
}
67+
@SuppressWarnings("unchecked")
68+
Map<String, ?> runtimeFieldMapping = (Map<String, ?>) runtimeFieldMappingObject;
69+
runtimeFieldMappingConsumer.accept(runtimeFieldName, runtimeFieldMapping);
70+
}
71+
}
5572
}

0 commit comments

Comments
 (0)