Merge branch 'main' into feature/93582-downsample-persistent-task

benchmarks/build.gradle

@@ -21,6 +21,7 @@ base {
 }
 
 tasks.named("test").configure { enabled = false }
+tasks.named("javadoc").configure { enabled = false }
 
 configurations {
   expression

docs/changelog/97700.yaml

@@ -0,0 +1,6 @@
+pr: 97700
+summary: Add `completion_time` time field to `async_search` get and status response
+area: Search
+type: enhancement
+issues:
+ - 88640

docs/changelog/97703.yaml

@@ -0,0 +1,6 @@
+pr: 97703
+summary: Track `max_score` in collapse when requested
+area: Search
+type: bug
+issues:
+ - 97653

docs/changelog/97705.yaml

@@ -0,0 +1,7 @@
+pr: 97705
+summary: "Fix to stop aggregatable subobjects from being considered multi-fields, to support \
+  `\"subobjects\": false` in data frame analytics"
+area: Machine Learning
+type: bug
+issues:
+ - 88605

docs/changelog/97711.yaml

@@ -0,0 +1,5 @@
+pr: 97711
+summary: Release master service task on timeout
+area: Cluster Coordination
+type: bug
+issues: []

docs/reference/eql/eql.asciidoc

@@ -32,7 +32,7 @@ describe activity that goes beyond IOCs.
 
 With the exception of sample queries, EQL searches require that the searched
 data stream or index  contains a _timestamp_ field. By default, EQL uses the
-`@timestamp` field from the {ecs-ref}[Elastic Common Schema (ECS)]. 
+`@timestamp` field from the {ecs-ref}[Elastic Common Schema (ECS)].
 
 EQL searches also require an _event category_ field, unless you use the
 <<eql-syntax-match-any-event-category,`any` keyword>> to search for  documents
@@ -555,9 +555,9 @@ GET /my-index*/_eql/search
 {
   "query": """
     sample by host
-      [any where uptime > 0] 
-      [any where port > 100] 
-      [any where bool == true] 
+      [any where uptime > 0]
+      [any where port > 100]
+      [any where bool == true]
   """
 }
 ----
@@ -691,7 +691,7 @@ each of the filters. Events are returned in the order of the filters they match:
   }
 }
 ----
-// TESTRESPONSE[s/  \.\.\.\n/"is_partial": false, "is_running": false, "took": $body.took, "timed_out": false,/]
+// TESTRESPONSE[skip:"AwaitsFix https://github.com/elastic/elasticsearch/issues/97646]
 
 <1> The events in the first sample have a value of `doom` for `host`.
 <2> This event matches the first filter.
@@ -854,7 +854,7 @@ By default, the response of a sample query contains up to 10 samples, with one
 sample per unique set of join keys. Use the `size` parameter to get a smaller or
 larger set of samples. To retrieve more than one sample per set of join keys,
 use the `max_samples_per_key` parameter. Pipes are not supported for sample
-queries. 
+queries.
 
 [source,console]
 ----

docs/reference/search/async-search.asciidoc

@@ -70,6 +70,7 @@ search results are returned as part of the
 // TESTRESPONSE[s/"is_running" : true/"is_running": $body.is_running/]
 // TESTRESPONSE[s/1583945890986/$body.start_time_in_millis/]
 // TESTRESPONSE[s/1584377890986/$body.expiration_time_in_millis/]
+// TESTRESPONSE[s/"response"/"completion_time_in_millis": $body.completion_time_in_millis,\n  "response"/]
 // TESTRESPONSE[s/"took" : 1122/"took": $body.response.took/]
 // TESTRESPONSE[s/"num_reduce_phases" : 0,//]
 // TESTRESPONSE[s/"total" : 562/"total": $body.response._shards.total/]
@@ -156,17 +157,18 @@ GET /_async_search/FmRldE8zREVEUzA2ZVpUeGs2ejJFUFEaMkZ5QTVrSTZSaVN3WlNFVmtlWHJsd
 --------------------------------------------------
 {
   "id" : "FmRldE8zREVEUzA2ZVpUeGs2ejJFUFEaMkZ5QTVrSTZSaVN3WlNFVmtlWHJsdzoxMDc=",
-  "is_partial" : true, <1>
-  "is_running" : true, <2>
+  "is_partial" : false, <1>
+  "is_running" : false, <2>
   "start_time_in_millis" : 1583945890986,
   "expiration_time_in_millis" : 1584377890986, <3>
+  "completion_time_in_millis" : 1583945903130, <4>
   "response" : {
     "took" : 12144,
     "timed_out" : false,
-    "num_reduce_phases" : 46, <4>
+    "num_reduce_phases" : 46, <5>
     "_shards" : {
       "total" : 562,
-      "successful" : 188, <5>
+      "successful" : 188, <6>
       "skipped" : 0,
       "failed" : 0
     },
@@ -178,7 +180,7 @@ GET /_async_search/FmRldE8zREVEUzA2ZVpUeGs2ejJFUFEaMkZ5QTVrSTZSaVN3WlNFVmtlWHJsd
       "max_score" : null,
       "hits" : [ ]
     },
-    "aggregations" : { <6>
+    "aggregations" : { <7>
       "sale_date" :  {
         "buckets" : []
       }
@@ -191,6 +193,7 @@ GET /_async_search/FmRldE8zREVEUzA2ZVpUeGs2ejJFUFEaMkZ5QTVrSTZSaVN3WlNFVmtlWHJsd
 // TESTRESPONSE[s/"is_running" : true/"is_running" : false/]
 // TESTRESPONSE[s/1583945890986/$body.start_time_in_millis/]
 // TESTRESPONSE[s/1584377890986/$body.expiration_time_in_millis/]
+// TESTRESPONSE[s/1583945903130/$body.completion_time_in_millis/]
 // TESTRESPONSE[s/"took" : 12144/"took": $body.response.took/]
 // TESTRESPONSE[s/"total" : 562/"total": $body.response._shards.total/]
 // TESTRESPONSE[s/"successful" : 188/"successful": $body.response._shards.successful/]
@@ -203,13 +206,14 @@ or was successfully completed on all shards. While the query is being
 executed, `is_partial` is always set to `true`
 <2> Whether the search is still being executed or it has completed
 <3> When the async search will expire
-<4> Indicates how many reductions of the results have been performed. If this
+<4> When the async search has finished, the completion_time is indicated (start_time + took)
+<5> Indicates how many reductions of the results have been performed. If this
 number increases compared to the last retrieved results, you can expect
 additional results included in the search response
-<5> Indicates how many shards have executed the query. Note that in order for
+<6> Indicates how many shards have executed the query. Note that in order for
 shard results to be included in the search response, they need to be reduced
 first.
-<6> Partial aggregations results, coming from the shards that have already
+<7> Partial aggregations results, coming from the shards that have already
 completed the execution of the query.
 
 The `wait_for_completion_timeout` parameter can also be provided when calling

docs/reference/search/search-your-data/collapse-search-results.asciidoc

@@ -267,7 +267,7 @@ GET /my-index-000001/_search
                 "value" : 1,
                 "relation" : "eq"
               },
-              "max_score" : null,
+              "max_score" : 0.5753642,
               "hits" : [
                 {
                   "_index" : "my-index-000001",
@@ -311,3 +311,9 @@ GET /my-index-000001/_search
   }
 }
 ----
+
+[discrete]
+=== Track Scores
+
+When `collapse` is used with `sort` on a field, scores are not computed.
+Setting `track_scores` to true instructs {es} to compute and track scores.

docs/reference/search/search-your-data/search-across-clusters.asciidoc

@@ -365,6 +365,7 @@ The API returns the following response:
 // TESTRESPONSE[s/"is_running": true/"is_running": $body.is_running/]
 // TESTRESPONSE[s/1685563581380/$body.start_time_in_millis/]
 // TESTRESPONSE[s/1685995581380/$body.expiration_time_in_millis/]
+// TESTRESPONSE[s/"response"/"completion_time_in_millis": $body.completion_time_in_millis,\n  "response"/]
 // TESTRESPONSE[s/"num_reduce_phases": 0/"num_reduce_phases": "$body.response.num_reduce_phases"/]
 // TESTRESPONSE[s/"took": 1020/"took": "$body.response.took"/]
 // TESTRESPONSE[s/"total": 8/"total": $body.response._shards.total/]
@@ -465,19 +466,20 @@ Response:
   "is_running": false,
   "start_time_in_millis": 1685564911108,
   "expiration_time_in_millis": 1685996911108,
+  "completion_time_in_millis": 1685564938727,  <1>
   "response": {
     "took": 27619,
     "timed_out": false,
     "num_reduce_phases": 4,
     "_shards": {
       "total": 28,
-      "successful": 28,  <1>
+      "successful": 28,  <2>
       "skipped": 0,
       "failed": 0
     },
     "_clusters": {
       "total": 3,
-      "successful": 3,   <2>
+      "successful": 3,   <3>
       "skipped": 0
     },
     "hits": {
@@ -496,6 +498,7 @@ Response:
 // TESTRESPONSE[s/"is_running": true/"is_running": $body.is_running/]
 // TESTRESPONSE[s/1685564911108/$body.start_time_in_millis/]
 // TESTRESPONSE[s/1685996911108/$body.expiration_time_in_millis/]
+// TESTRESPONSE[s/1685564938727/$body.completion_time_in_millis/]
 // TESTRESPONSE[s/"took": 27619/"took": "$body.response.took"/]
 // TESTRESPONSE[s/"total": 28/"total": $body.response._shards.total/]
 // TESTRESPONSE[s/"successful": 28/"successful": $body.response._shards.successful/]
@@ -506,9 +509,10 @@ Response:
 // TESTRESPONSE[s/"hits": \[...list of hits here...\]/"hits": $body.response.hits.hits/]
 
 
-<1> The `_shards` section is now updated to show that 28 total shards
+<1> Once the search has finished, the completion_time is present.
+<2> The `_shards` section is now updated to show that 28 total shards
 were searched across all clusters and that all were successful.
-<2> The `_clusters` section shows that searches on all 3 clusters were successful.
+<3> The `_clusters` section shows that searches on all 3 clusters were successful.
 
 
 
@@ -592,6 +596,7 @@ the `wait_for_completion_timeout` duration (see <<async-search>>).
 // TESTRESPONSE[s/"is_running": true/"is_running": $body.is_running/]
 // TESTRESPONSE[s/1685563581380/$body.start_time_in_millis/]
 // TESTRESPONSE[s/1685995581380/$body.expiration_time_in_millis/]
+// TESTRESPONSE[s/"response"/"completion_time_in_millis": $body.completion_time_in_millis,\n  "response"/]
 // TESTRESPONSE[s/"num_reduce_phases": 0/"num_reduce_phases": "$body.response.num_reduce_phases"/]
 // TESTRESPONSE[s/"took": 1020/"took": "$body.response.took"/]
 // TESTRESPONSE[s/"total": 28/"total": $body.response._shards.total/]

docs/reference/settings/data-stream-lifecycle-settings.asciidoc

@@ -15,7 +15,7 @@ These are the settings available for configuring <<data-stream-lifecycle, data s
 [[data-streams-lifecycle-poll-interval]]
 `data_streams.lifecycle.poll_interval`::
 (<<dynamic-cluster-setting,Dynamic>>, <<time-units, time unit value>>)
-How often {es} checks what is the next action for all data streams with a built-in lifecycle. Defaults to `10m`.
+How often {es} checks what is the next action for all data streams with a built-in lifecycle. Defaults to `5m`.
 
 [[cluster-lifecycle-default-rollover]]
 `cluster.lifecycle.default.rollover`::

modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java

@@ -77,7 +77,7 @@ public class DataStreamLifecycleService implements ClusterStateListener, Closeab
     public static final String DATA_STREAM_LIFECYCLE_POLL_INTERVAL = "data_streams.lifecycle.poll_interval";
     public static final Setting<TimeValue> DATA_STREAM_LIFECYCLE_POLL_INTERVAL_SETTING = Setting.timeSetting(
         DATA_STREAM_LIFECYCLE_POLL_INTERVAL,
-        TimeValue.timeValueMinutes(10),
+        TimeValue.timeValueMinutes(5),
         TimeValue.timeValueSeconds(1),
         Setting.Property.Dynamic,
         Setting.Property.NodeScope

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/111_field_collapsing_with_max_score.yml

@@ -0,0 +1,113 @@
+setup:
+  - skip:
+      version: " - 8.9.99"
+      reason: Collapse with max score was fixed in 8.10.0
+      features: close_to
+
+  - do:
+      indices.create:
+        index: user_comments
+        body:
+          settings:
+            index:
+              number_of_shards: 1
+          mappings:
+              properties:
+                user_id: { type: keyword }
+                comment: { type: text }
+
+  - do:
+      bulk:
+        index: user_comments
+        refresh: true
+        body:
+          - '{"index": {"_id": "1"}}'
+          - '{"user_id": "user1", "comment": "Canada is beautiful."}'
+          - '{"index": {"_id": "2"}}'
+          - '{"user_id": "user1", "comment": "Canada is the second-largest country in the world by land area."}'
+          - '{"index": {"_id": "3"}}'
+          - '{"user_id": "user2", "comment": "The capital of Canada is Ottawa."}'
+          - '{"index": {"_id": "4"}}'
+          - '{"user_id": "user2", "comment": "Canada is cold."}'
+          - '{"index": {"_id": "5"}}'
+          - '{"user_id": "user3", "comment": "Canada celebrates Canada Day on July 1st."}'
+
+---
+"Max score returned when there is no sort":
+  - do:
+      search:
+        index: user_comments
+        body:
+          query:
+            match:
+              comment: "Canada"
+          collapse:
+            field: user_id
+
+  - match: {hits.total.value: 5 }
+  - length: {hits.hits: 3 }
+  - match: {hits.hits.0.fields.user_id: ["user3"] }
+  - close_to: { hits.max_score: { value: 0.11545, error: 0.00001 } }
+
+---
+"Max score is not returned when there is sort and track_scores is false (by default)":
+  - do:
+      search:
+        index: user_comments
+        body:
+          sort:
+            user_id: asc
+          query:
+            match:
+              comment: "Canada"
+          collapse:
+            field: user_id
+
+  - match: {hits.total.value: 5 }
+  - length: {hits.hits: 3 }
+  - match: {hits.hits.0.fields.user_id: ["user1"] }
+  - is_false: hits.max_score
+
+---
+"Max score returned when there is sort and track_scores is true":
+  - do:
+      search:
+        index: user_comments
+        body:
+          sort:
+            user_id: asc
+          query:
+            match:
+              comment: "Canada"
+          collapse:
+            field: user_id
+          track_scores: true
+
+  - match: {hits.total.value: 5 }
+  - length: {hits.hits: 3 }
+  - match: {hits.hits.0.fields.user_id: ["user1"] }
+  - close_to: { hits.max_score: { value: 0.11545, error: 0.00001 } }
+
+---
+"Max score returned over all documents not only top hits":
+  - do:
+      search:
+        index: user_comments
+        body:
+          size: 2
+          sort:
+            user_id: asc
+          query:
+            match:
+              comment: "Canada"
+          collapse:
+            field: user_id
+          track_scores: true
+
+  - match: {hits.total.value: 5 }
+  - length: {hits.hits: 2 }
+  - close_to: { hits.max_score: { value: 0.11545, error: 0.00001 } }
+  - match: {hits.hits.0.fields.user_id: ["user1"] }
+  - close_to: { hits.hits.0._score: { value: 0.11030, error: 0.00001 } }
+  - match: { hits.hits.1.fields.user_id: [ "user2" ] }
+  - close_to: { hits.hits.1._score: { value: 0.08817, error: 0.00001 } }

server/src/main/java/org/elasticsearch/TransportVersion.java

@@ -155,10 +155,15 @@ private static TransportVersion registerTransportVersion(int id, String uniqueId
     public static final TransportVersion V_8_500_032 = registerTransportVersion(8_500_032, "a9a14bc6-c3f2-41d9-a3d8-c686bf2c901d");
     public static final TransportVersion V_8_500_033 = registerTransportVersion(8_500_033, "193ab7c4-a751-4cbd-a66a-2d7d56ccbc10");
     public static final TransportVersion V_8_500_034 = registerTransportVersion(8_500_034, "16871c8b-88ba-4432-980a-10fd9ecad2dc");
-    public static final TransportVersion V_8_500_035 = registerTransportVersion(8_500_035, "ec065a44-b468-4f8a-aded-7b90ca8d792b");
+
+    public static final TransportVersion V_8_500_035 = registerTransportVersion(8_500_035, "664dd6ce-3487-4fbd-81a9-af778b28be45");
+
+    // Introduced for stateless plugin
+    public static final TransportVersion V_8_500_036 = registerTransportVersion(8_500_036, "3343c64f-d7ac-4f02-9262-3e1acfc56f89");
+    public static final TransportVersion V_8_500_037 = registerTransportVersion(8_500_035, "ec065a44-b468-4f8a-aded-7b90ca8d792b");
 
     private static class CurrentHolder {
-        private static final TransportVersion CURRENT = findCurrent(V_8_500_035);
+        private static final TransportVersion CURRENT = findCurrent(V_8_500_037);
 
         // finds the pluggable current version, or uses the given fallback
         private static TransportVersion findCurrent(TransportVersion fallback) {

server/src/main/java/org/elasticsearch/action/downsample/DownsampleConfig.java

@@ -111,7 +111,7 @@ public DownsampleConfig(final DateHistogramInterval fixedInterval, final TimeVal
 
     public DownsampleConfig(final StreamInput in) throws IOException {
         fixedInterval = new DateHistogramInterval(in);
-        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_500_035)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_500_037)) {
             timeout = TimeValue.parseTimeValue(in.readString(), TIMEOUT);
         } else {
             timeout = DEFAULT_TIMEOUT;
@@ -152,7 +152,7 @@ public static void validateSourceAndTargetIntervals(DownsampleConfig source, Dow
     @Override
     public void writeTo(final StreamOutput out) throws IOException {
         fixedInterval.writeTo(out);
-        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_500_035)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_500_037)) {
             out.writeString(timeout.getStringRep());
         } else {
             out.writeString(DEFAULT_TIMEOUT.getStringRep());