Move runtime fields stats to server

docs/reference/cluster/stats.asciidoc

@@ -492,6 +492,72 @@ Number of fields mapped to the field data type in selected nodes.
 `index_count`::
 (integer)
 Number of indices containing a mapping of the field data type in selected nodes.
+======
+
+`runtime_field_types`::
+(array of objects)
+Contains statistics about <<runtime-mapping-fields, runtime field data types>> used in selected
+nodes.
++
+.Properties of `runtime_field_types` objects
+[%collapsible%open]
+======
+`name`::
+(string)
+Field data type used in selected nodes.
+
+`count`::
+(integer)
+Number of runtime fields mapped to the field data type in selected nodes.
+
+`index_count`::
+(integer)
+Number of indices containing a mapping of the runtime field data type in selected nodes.
+
+`scriptless_count`::
+(integer)
+Number of runtime fields that don't declare a script.
+
+`shadowed_count`::
+(integer)
+Number of runtime fields that shadow an indexed field.
+
+`lang`::
+(array of strings)
+Script languages used for the runtime fields scripts
+
+`lines_max`::
+(integer)
+Maximum number of lines for a single runtime field script
+
+`lines_total`::
+(integer)
+Total number of lines for the scripts that define the current runtime field data type
+
+`chars_max`::
+(integer)
+Maximum number of characters for a single runtime field script
+
+`chars_total`::
+(integer)
+Total number of characters for the scripts that define the current runtime field data type
+
+`source_max`::
+(integer)
+Maximum number of accesses to _source for a single runtime field script
+
+`source_total`::
+(integer)
+Total number of accesses to _source for the scripts that define the current runtime field data type
+
+`doc_max`::
+(integer)
+Maximum number of accesses to doc_values for a single runtime field script
+
+`doc_total`::
+(integer)
+Total number of accesses to doc_values for the scripts that define the current runtime field data type
+
 ======
 =====
 
@@ -1220,7 +1286,8 @@ The API returns the following response:
          "file_sizes": {}
       },
       "mappings": {
-        "field_types": []
+        "field_types": [],
+        "runtime_field_types": []
       },
       "analysis": {
         "char_filter_types": [],
@@ -1363,6 +1430,7 @@ The API returns the following response:
 // TESTRESPONSE[s/"count": \{[^\}]*\}/"count": $body.$_path/]
 // TESTRESPONSE[s/"packaging_types": \[[^\]]*\]/"packaging_types": $body.$_path/]
 // TESTRESPONSE[s/"field_types": \[[^\]]*\]/"field_types": $body.$_path/]
+// TESTRESPONSE[s/"runtime_field_types": \[[^\]]*\]/"runtime_field_types": $body.$_path/]
 // TESTRESPONSE[s/: true|false/: $body.$_path/]
 // TESTRESPONSE[s/: (\-)?[0-9]+/: $body.$_path/]
 // TESTRESPONSE[s/: "[^"]*"/: $body.$_path/]

docs/reference/rest-api/info.asciidoc

@@ -114,10 +114,6 @@ Example response:
          "available": true,
          "enabled": true
       },
-      "runtime_fields": {
-         "available": true,
-         "enabled": true
-      },
       "searchable_snapshots" : {
          "available" : true,
          "enabled" : true

docs/reference/rest-api/usage.asciidoc

@@ -392,11 +392,6 @@ GET /_xpack/usage
   "aggregate_metric" : {
     "available" : true,
     "enabled" : true
-  },
-  "runtime_fields" : {
-    "available" : true,
-    "enabled" : true,
-    "field_types" : []
   }
 }
 ------------------------------------------------------------

rest-api-spec/src/main/resources/rest-api-spec/test/cluster.stats/10_basic.yml

@@ -83,3 +83,86 @@
       cluster.stats: {}
 
   - is_true: nodes.packaging_types
+---
+"get cluster stats without runtime fields":
+  - skip:
+      version: " - 7.99.99"
+      reason:  "cluster stats includes runtime fields from 8.0 on"
+  - do:
+      indices.create:
+        index: sensor
+
+  - do: {cluster.stats: {}}
+  - length: { indices.mappings.field_types: 0 }
+  - length: { indices.mappings.runtime_field_types: 0 }
+
+---
+"Usage stats with script-less runtime fields":
+  - skip:
+      version: " - 7.99.99"
+      reason:  "cluster stats includes runtime fields from 8.0 on"
+  - do:
+      indices.create:
+        index: sensor
+        body:
+          mappings:
+            runtime:
+              message_from_source:
+                type: keyword
+              bad_map:
+                type: double    # shadows the bad_map field in properties
+              message.text:
+                type: keyword   # shadows the message.text subfield in properties
+            properties:
+              message:
+                type: keyword
+                fields:
+                  text:
+                    type: text
+              bad_map:
+                type: long
+
+  - do: {cluster.stats: {}}
+  - length: { indices.mappings.field_types: 3 }
+
+  - match: { indices.mappings.field_types.0.name: keyword }
+  - match: { indices.mappings.field_types.0.count: 1 }
+  - match: { indices.mappings.field_types.0.index_count: 1 }
+  - match: { indices.mappings.field_types.1.name: long }
+  - match: { indices.mappings.field_types.1.count: 1 }
+  - match: { indices.mappings.field_types.1.index_count: 1 }
+  - match: { indices.mappings.field_types.2.name: text }
+  - match: { indices.mappings.field_types.2.count: 1 }
+  - match: { indices.mappings.field_types.2.index_count: 1 }
+
+
+  - length: { indices.mappings.runtime_field_types: 2 }
+
+  - match: { indices.mappings.runtime_field_types.0.name: double }
+  - match: { indices.mappings.runtime_field_types.0.count: 1 }
+  - match: { indices.mappings.runtime_field_types.0.index_count: 1 }
+  - match: { indices.mappings.runtime_field_types.0.scriptless_count: 1 }
+  - match: { indices.mappings.runtime_field_types.0.shadowed_count: 1 }
+  - match: { indices.mappings.runtime_field_types.0.source_max: 0 }
+  - match: { indices.mappings.runtime_field_types.0.source_total: 0 }
+  - match: { indices.mappings.runtime_field_types.0.lines_max:  0 }
+  - match: { indices.mappings.runtime_field_types.0.lines_total: 0 }
+  - match: { indices.mappings.runtime_field_types.0.chars_max:  0 }
+  - match: { indices.mappings.runtime_field_types.0.chars_total: 0 }
+  - match: { indices.mappings.runtime_field_types.0.doc_max: 0 }
+  - match: { indices.mappings.runtime_field_types.0.doc_total: 0 }
+
+  - match: { indices.mappings.runtime_field_types.1.name: keyword }
+  - match: { indices.mappings.runtime_field_types.1.count: 2 }
+  - match: { indices.mappings.runtime_field_types.1.index_count: 1 }
+  - match: { indices.mappings.runtime_field_types.1.scriptless_count: 2 }
+  - match: { indices.mappings.runtime_field_types.1.shadowed_count: 1 }
+  - match: { indices.mappings.runtime_field_types.1.source_max: 0 }
+  - match: { indices.mappings.runtime_field_types.1.source_total: 0 }
+  - match: { indices.mappings.runtime_field_types.1.lines_max:  0 }
+  - match: { indices.mappings.runtime_field_types.1.lines_total: 0 }
+  - match: { indices.mappings.runtime_field_types.1.chars_max:  0 }
+  - match: { indices.mappings.runtime_field_types.1.chars_total: 0 }
+  - match: { indices.mappings.runtime_field_types.1.doc_max: 0 }
+  - match: { indices.mappings.runtime_field_types.1.doc_total: 0 }
+

server/src/main/java/org/elasticsearch/action/admin/cluster/stats/MappingStats.java

@@ -8,6 +8,7 @@
 
 package org.elasticsearch.action.admin.cluster.stats;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.MappingMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
@@ -28,7 +29,10 @@
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 /**
  * Usage statistics about mappings usage.
@@ -40,6 +44,8 @@ public final class MappingStats implements ToXContentFragment, Writeable {
      */
     public static MappingStats of(Metadata metadata, Runnable ensureNotCancelled) {
         Map<String, IndexFeatureStats> fieldTypes = new HashMap<>();
+        Set<String> concreteFieldNames = new HashSet<>();
+        Map<String, RuntimeFieldStats> runtimeFieldTypes = new HashMap<>();
         for (IndexMetadata indexMetadata : metadata) {
             ensureNotCancelled.run();
             if (indexMetadata.isSystem()) {
@@ -48,9 +54,11 @@ public static MappingStats of(Metadata metadata, Runnable ensureNotCancelled) {
                 continue;
             }
             Set<String> indexFieldTypes = new HashSet<>();
+            Set<String> indexRuntimeFieldTypes = new HashSet<>();
             MappingMetadata mappingMetadata = indexMetadata.mapping();
             if (mappingMetadata != null) {
                 MappingVisitor.visitMapping(mappingMetadata.getSourceAsMap(), (field, fieldMapping) -> {
+                    concreteFieldNames.add(field);
                     String type = null;
                     Object typeO = fieldMapping.get("type");
                     if (typeO != null) {
@@ -67,26 +75,83 @@ public static MappingStats of(Metadata metadata, Runnable ensureNotCancelled) {
                         }
                     }
                 });
+
+                MappingVisitor.visitRuntimeMapping(mappingMetadata.getSourceAsMap(), (field, fieldMapping) -> {
+                    Object typeObject = fieldMapping.get("type");
+                    if (typeObject == null) {
+                        return;
+                    }
+                    String type = typeObject.toString();
+                    RuntimeFieldStats stats = runtimeFieldTypes.computeIfAbsent(type, RuntimeFieldStats::new);
+                    stats.count++;
+                    if (indexRuntimeFieldTypes.add(type)) {
+                        stats.indexCount++;
+                    }
+                    if (concreteFieldNames.contains(field)) {
+                        stats.shadowedCount++;
+                    }
+                    Object scriptObject = fieldMapping.get("script");
+                    if (scriptObject == null) {
+                        stats.scriptLessCount++;
+                    } else if (scriptObject instanceof Map) {
+                        Map<?, ?> script = (Map<?, ?>) scriptObject;
+                        Object sourceObject = script.get("source");
+                        if (sourceObject != null) {
+                            String scriptSource = sourceObject.toString();
+                            int chars = scriptSource.length();
+                            long lines = scriptSource.lines().count();
+                            int docUsages = countOccurrences(scriptSource, "doc[\\[\\.]");
+                            int sourceUsages = countOccurrences(scriptSource, "params\\._source");
+                            stats.update(chars, lines, sourceUsages, docUsages);
+                        }
+                        Object langObject = script.get("lang");
+                        if (langObject != null) {
+                            stats.scriptLangs.add(langObject.toString());
+                        }
+                    }
+                });
             }
         }
-        return new MappingStats(fieldTypes.values());
+        return new MappingStats(fieldTypes.values(), runtimeFieldTypes.values());
+    }
+
+    private static int countOccurrences(String script, String keyword) {
+        int occurrences = 0;
+        Pattern pattern = Pattern.compile(keyword);
+        Matcher matcher = pattern.matcher(script);
+        while (matcher.find()) {
+            occurrences++;
+        }
+        return occurrences;
     }
 
     private final Set<IndexFeatureStats> fieldTypeStats;
+    private final Set<RuntimeFieldStats> runtimeFieldTypeStats;
 
-    MappingStats(Collection<IndexFeatureStats> fieldTypeStats) {
+    MappingStats(Collection<IndexFeatureStats> fieldTypeStats, Collection<RuntimeFieldStats> runtimeFieldTypeStats) {
         List<IndexFeatureStats> stats = new ArrayList<>(fieldTypeStats);
         stats.sort(Comparator.comparing(IndexFeatureStats::getName));
         this.fieldTypeStats = Collections.unmodifiableSet(new LinkedHashSet<IndexFeatureStats>(stats));
+        List<RuntimeFieldStats> runtimeStats = new ArrayList<>(runtimeFieldTypeStats);
+        runtimeStats.sort(Comparator.comparing(RuntimeFieldStats::type));
+        this.runtimeFieldTypeStats = Collections.unmodifiableSet(new LinkedHashSet<>(runtimeStats));
     }
 
     MappingStats(StreamInput in) throws IOException {
         fieldTypeStats = Collections.unmodifiableSet(new LinkedHashSet<>(in.readList(IndexFeatureStats::new)));
+        if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
+            runtimeFieldTypeStats = Collections.unmodifiableSet(new LinkedHashSet<>(in.readList(RuntimeFieldStats::new)));
+        } else {
+            runtimeFieldTypeStats = Collections.emptySet();
+        }
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeCollection(fieldTypeStats);
+        if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+            out.writeCollection(runtimeFieldTypeStats);
+        }
     }
 
     /**
@@ -96,6 +161,13 @@ public Set<IndexFeatureStats> getFieldTypeStats() {
         return fieldTypeStats;
     }
 
+    /**
+     * Return stats about runtime field types.
+     */
+    public Set<RuntimeFieldStats> getRuntimeFieldTypeStats() {
+        return runtimeFieldTypeStats;
+    }
+
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject("mappings");
@@ -104,6 +176,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             st.toXContent(builder, params);
         }
         builder.endArray();
+        builder.startArray("runtime_field_types");
+        for (RuntimeFieldStats st : runtimeFieldTypeStats) {
+            st.toXContent(builder, params);
+        }
+        builder.endArray();
         builder.endObject();
         return builder;
     }
@@ -119,11 +196,12 @@ public boolean equals(Object o) {
             return false;
         }
         MappingStats that = (MappingStats) o;
-        return fieldTypeStats.equals(that.fieldTypeStats);
+        return fieldTypeStats.equals(that.fieldTypeStats) &&
+            runtimeFieldTypeStats.equals(that.runtimeFieldTypeStats);
     }
 
     @Override
     public int hashCode() {
-        return fieldTypeStats.hashCode();
+        return Objects.hash(fieldTypeStats, runtimeFieldTypeStats);
     }
 }

server/src/main/java/org/elasticsearch/action/admin/cluster/stats/MappingVisitor.java

@@ -52,4 +52,21 @@ private static void visitMapping(Map<String, ?> mapping, String path, BiConsumer
         }
     }
 
+    public static void visitRuntimeMapping(Map<String, ?> mapping, BiConsumer<String, Map<String, ?>> runtimeFieldMappingConsumer) {
+        Object runtimeObject = mapping.get("runtime");
+        if (runtimeObject instanceof Map == false) {
+            return;
+        }
+        @SuppressWarnings("unchecked")
+        Map<String, ?> runtimeMappings = (Map<String, ?>) runtimeObject;
+        for (String runtimeFieldName : runtimeMappings.keySet()) {
+            Object runtimeFieldMappingObject = runtimeMappings.get(runtimeFieldName);
+            if (runtimeFieldMappingObject instanceof Map == false) {
+                continue;
+            }
+            @SuppressWarnings("unchecked")
+            Map<String, ?> runtimeFieldMapping = (Map<String, ?>) runtimeFieldMappingObject;
+            runtimeFieldMappingConsumer.accept(runtimeFieldName, runtimeFieldMapping);
+        }
+    }
 }