Add field type for version strings (#59773)

docs/reference/mapping/types.asciidoc

@@ -51,6 +51,8 @@ Dates::                 Date types, including <<date,`date`>> and
 <<range,Range>>::   Range types, such as `long_range`, `double_range`,
                     `date_range`, and `ip_range`.
 <<ip,`ip`>>::       IPv4 and IPv6 addresses.
+<<version,Version>>::  Software versions. Supports https://semver.org/[Semantic Versioning]
+precedence rules.
 {plugins}/mapper-murmur3.html[`murmur3`]:: Compute and stores hashes of
 values.
 
@@ -148,6 +150,8 @@ include::types/geo-shape.asciidoc[]
 
 include::types/ip.asciidoc[]
 
+include::types/version.asciidoc[]
+
 include::types/parent-join.asciidoc[]
 
 include::types/keyword.asciidoc[]

docs/reference/mapping/types/keyword.asciidoc

@@ -134,3 +134,4 @@ The following parameters are accepted by `keyword` fields:
 include::constant-keyword.asciidoc[]
 
 include::wildcard.asciidoc[]
+

docs/reference/mapping/types/version.asciidoc

@@ -0,0 +1,70 @@
+[role="xpack"]
+[testenv="basic"]
+[[version]]
+=== Version field type
+++++
+<titleabbrev>Version</titleabbrev>
+++++
+
+The `version` field type is a specialization of the `keyword` field for
+handling software version values and to support specialized precedence
+rules for them. Precedence is defined following the rules outlined by
+https://semver.org/[Semantic Versioning], which for example means that
+major, minor and patch version parts are sorted numerically (i.e. 
+"2.1.0" < "2.4.1" < "2.11.2") and pre-release versions are sorted before
+release versions (i.e. "1.0.0-alpha" < "1.0.0").
+
+You index a `version` field as follows
+
+[source,console]
+--------------------------------------------------
+PUT my-index-000001
+{
+  "mappings": {
+    "properties": {
+      "my_version": {
+        "type": "version"
+      }
+    }
+  }
+}
+
+--------------------------------------------------
+
+The field offers the same search capabilities as a regular keyword field. It 
+can e.g. be searched for exact matches using `match` or `term` queries and
+supports prefix and wildcard searches. The main benefit is that `range` queries
+will honor Semver ordering, so a `range` query between "1.0.0" and "1.5.0"
+will include versions of "1.2.3" but not "1.11.2" for example. Note that this
+would be different when using a regular `keyword` field for indexing where ordering
+is alphabetical.
+
+Software versions are expected to follow the
+https://semver.org/[Semantic Versioning rules] schema and precedence rules with
+the notable exception that more or less than three main version identifiers are
+allowed (i.e. "1.2" or "1.2.3.4" qualify as valid versions while they wouldn't under
+strict Semver rules). Version strings that are not valid under the Semver definition
+(e.g. "1.2.alpha.4") can still be indexed and retrieved as exact matches, however they
+will all appear _after_ any valid version with regular alphabetical ordering. The empty
+String "" is considered invalid and sorted after all valid versions, but before other
+invalid ones.
+
+[discrete]
+[[version-params]]
+==== Parameters for version fields
+
+The following parameters are accepted by `version` fields:
+
+[horizontal]
+
+<<mapping-field-meta,`meta`>>::
+
+    Metadata about the field.
+
+[discrete]
+==== Limitations
+
+This field type isn't optimized for heavy wildcard, regex or fuzzy searches. While those
+type of queries work in this field, you should consider using a regular `keyword` field if
+you strongly rely on these kind of queries.
+

server/src/main/java/org/elasticsearch/index/mapper/TermBasedFieldType.java

@@ -33,9 +33,10 @@
 
 /** Base {@link MappedFieldType} implementation for a field that is indexed
  *  with the inverted index. */
-abstract class TermBasedFieldType extends SimpleMappedFieldType {
+public abstract class TermBasedFieldType extends SimpleMappedFieldType {
 
-    TermBasedFieldType(String name, boolean isSearchable, boolean hasDocValues, TextSearchInfo textSearchInfo, Map<String, String> meta) {
+    public TermBasedFieldType(String name, boolean isSearchable, boolean hasDocValues, TextSearchInfo textSearchInfo,
+        Map<String, String> meta) {
         super(name, isSearchable, hasDocValues, textSearchInfo, meta);
     }
 

x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregator.java

@@ -101,7 +101,7 @@ public void collect(int doc, long bucket) throws IOException {
                     for (int i = 0; i < valuesCount; i++) {
                         BytesRef value = values.nextValue();
                         if (value.length > 0) {
-                            String valueStr = value.utf8ToString();
+                            String valueStr = (String) format.format(value);
                             int length = valueStr.length();
                             totalLength.increment(bucket, length);
 

x-pack/plugin/src/test/resources/rest-api-spec/test/versionfield/10_basic.yml

@@ -0,0 +1,107 @@
+# Integration tests for the version field
+#
+---
+setup:
+
+  - skip:
+      features: headers
+      version: " - 7.99.99"
+      reason: "version field is added to 8.0 first"
+
+  - do:
+        indices.create:
+          index: test_index
+          body:
+              mappings:
+                properties:
+                  version:
+                    type: version
+
+  - do:
+        bulk:
+          refresh: true
+          body:
+            - '{ "index" : { "_index" : "test_index", "_id" : "1" } }'
+            - '{"version": "1.1.0" }'
+            - '{ "index" : { "_index" : "test_index", "_id" : "2" } }'
+            - '{"version": "2.0.0-beta" }'
+            - '{ "index" : { "_index" : "test_index", "_id" : "3" } }'
+            - '{"version": "3.1.0" }'
+
+---
+"Store malformed":
+  - do:
+        indices.create:
+          index: test_malformed
+          body:
+              mappings:
+                properties:
+                  version:
+                    type: version
+
+  - do:
+        bulk:
+          refresh: true
+          body:
+            - '{ "index" : { "_index" : "test_malformed", "_id" : "1" } }'
+            - '{"version": "1.1.0" }'
+            - '{ "index" : { "_index" : "test_malformed", "_id" : "2" } }'
+            - '{"version": "2.0.0-beta" }'
+            - '{ "index" : { "_index" : "test_malformed", "_id" : "3" } }'
+            - '{"version": "v3.1.0" }'
+            - '{ "index" : { "_index" : "test_malformed", "_id" : "4" } }'
+            - '{"version": "1.el6" }'
+
+  - do:
+        search:
+          index: test_malformed
+          body:
+            query: { "match" : { "version" : "1.el6" } }
+
+  - do:
+        search:
+          index: test_malformed
+          body:
+            query: { "match_all" : { } }
+            sort:
+              version: asc
+
+  - match: { hits.total.value: 4 }
+  - match: { hits.hits.0._source.version: "1.1.0" }
+  - match: { hits.hits.1._source.version: "2.0.0-beta" }
+  - match: { hits.hits.2._source.version: "1.el6" }
+  - match: { hits.hits.3._source.version: "v3.1.0" }
+
+---
+"Basic ranges":
+  - do:
+        search:
+          index: test_index
+          body:
+            query: { "range" : { "version" : { "gt" : "1.1.0", "lt" : "9999" } } }
+
+  - match: { hits.total.value: 2 }
+
+  - do:
+        search:
+          index: test_index
+          body:
+            query: { "range" : { "version" : { "gte" : "1.1.0", "lt" : "9999" } } }
+
+  - match: { hits.total.value: 3 }
+
+  - do:
+        search:
+          index: test_index
+          body:
+            query: { "range" : { "version" : { "gte" : "2.0.0", "lt" : "9999" } } }
+
+  - match: { hits.total.value: 1 }
+
+  - do:
+        search:
+          index: test_index
+          body:
+            query: { "range" : { "version" : { "gte" : "2.0.0-alpha", "lt" : "9999" } } }
+
+  - match: { hits.total.value: 2 }

x-pack/plugin/src/test/resources/rest-api-spec/test/versionfield/20_scripts.yml

@@ -0,0 +1,54 @@
+# Integration tests for the version field
+#
+---
+setup:
+
+  - skip:
+      features: headers
+      version: " - 7.99.99"
+      reason: "version field is added to 8.0 first"
+
+  - do:
+        indices.create:
+          index: test_index
+          body:
+              mappings:
+                properties:
+                  version:
+                    type: version
+
+  - do:
+        bulk:
+          refresh: true
+          body:
+            - '{ "index" : { "_index" : "test_index", "_id" : "1" } }'
+            - '{"version": "1.1.12" }'
+            - '{ "index" : { "_index" : "test_index", "_id" : "2" } }'
+            - '{"version": "2.0.0-beta" }'
+            - '{ "index" : { "_index" : "test_index", "_id" : "3" } }'
+            - '{"version": "3.1.0" }'
+
+---
+"Filter script":
+  - do:
+        search:
+          index: test_index
+          body:
+            query: { "script" : { "script" : { "source": "doc['version'].value.length() > 5"} } }
+
+  - match: { hits.total.value: 2 }
+  - match: { hits.hits.0._source.version: "1.1.12" }
+  - match: { hits.hits.1._source.version: "2.0.0-beta" }
+
+---
+"Sort script":
+  - do:
+        search:
+          index: test_index
+          body:
+            sort: { "_script" : { "type" : "number", "script" : { "source": "doc['version'].value.length()" } } }
+
+  - match: { hits.total.value: 3 }
+  - match: { hits.hits.0._source.version: "3.1.0" }
+  - match: { hits.hits.1._source.version: "1.1.12" }
+  - match: { hits.hits.2._source.version: "2.0.0-beta" }

x-pack/plugin/versionfield/build.gradle

@@ -0,0 +1,23 @@
+evaluationDependsOn(xpackModule('core'))
+
+apply plugin: 'elasticsearch.esplugin'
+apply plugin: 'elasticsearch.internal-cluster-test'
+
+esplugin {
+  name 'versionfield'
+  description 'A plugin for a field type to store sofware versions'
+  classname 'org.elasticsearch.xpack.versionfield.VersionFieldPlugin'
+  extendedPlugins = ['x-pack-core', 'lang-painless']
+}
+archivesBaseName = 'x-pack-versionfield'
+
+dependencies {
+  compileOnly project(path: xpackModule('core'), configuration: 'default')
+  compileOnly project(':modules:lang-painless:spi')
+  compileOnly(project(':modules:lang-painless')) {
+      // exclude ASM to not affect featureAware task on Java 10+
+      exclude group: "org.ow2.asm"
+  }
+  testImplementation project(path: xpackModule('core'), configuration: 'testArtifacts')
+  testImplementation project(path: xpackModule('analytics'), configuration: 'default')
+}

x-pack/plugin/versionfield/src/internalClusterTest/java/org/elasticsearch/xpack/versionfield/VersionFieldIT.java

@@ -0,0 +1,89 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.versionfield;
+
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.search.aggregations.AggregationBuilders;
+import org.elasticsearch.search.aggregations.bucket.terms.Terms;
+import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
+
+import java.util.Collection;
+import java.util.List;
+
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+
+public class VersionFieldIT extends ESIntegTestCase {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return org.elasticsearch.common.collect.List.of(VersionFieldPlugin.class, LocalStateCompositeXPackPlugin.class);
+    }
+
+    public void testTermsAggregation() throws Exception {
+        String indexName = "test";
+        createIndex(indexName);
+
+        client().admin()
+            .indices()
+            .preparePutMapping(indexName)
+            .setType("_doc")
+            .setSource(
+                XContentFactory.jsonBuilder()
+                    .startObject()
+                    .startObject("_doc")
+                    .startObject("properties")
+                    .startObject("version")
+                    .field("type", "version")
+                    .endObject()
+                    .endObject()
+                    .endObject()
+                    .endObject()
+            )
+            .get();
+        ensureGreen();
+
+        client().prepareIndex(indexName, "_doc")
+            .setId("1")
+            .setSource(jsonBuilder().startObject().field("version", "1.0").endObject())
+            .get();
+        client().prepareIndex(indexName, "_doc")
+            .setId("2")
+            .setSource(jsonBuilder().startObject().field("version", "1.3.0").endObject())
+            .get();
+        client().prepareIndex(indexName, "_doc")
+            .setId("3")
+            .setSource(jsonBuilder().startObject().field("version", "2.1.0-alpha").endObject())
+            .get();
+        client().prepareIndex(indexName, "_doc")
+            .setId("4")
+            .setSource(jsonBuilder().startObject().field("version", "2.1.0").endObject())
+            .get();
+        client().prepareIndex(indexName, "_doc")
+            .setId("5")
+            .setSource(jsonBuilder().startObject().field("version", "3.11.5").endObject())
+            .get();
+        refresh();
+
+        // terms aggs
+        SearchResponse response = client().prepareSearch(indexName)
+            .addAggregation(AggregationBuilders.terms("myterms").field("version"))
+            .get();
+        Terms terms = response.getAggregations().get("myterms");
+        List<? extends Bucket> buckets = terms.getBuckets();
+
+        assertEquals(5, buckets.size());
+        assertEquals("1.0", buckets.get(0).getKey());
+        assertEquals("1.3.0", buckets.get(1).getKey());
+        assertEquals("2.1.0-alpha", buckets.get(2).getKey());
+        assertEquals("2.1.0", buckets.get(3).getKey());
+        assertEquals("3.11.5", buckets.get(4).getKey());
+    }
+}