Add Fetch Phase to Search Profile

CHANGELOG.md

@@ -26,6 +26,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Extend Approximation Framework to other numeric types ([#18530](https://github.com/opensearch-project/OpenSearch/issues/18530))
 - Add Semantic Version field type mapper and extensive unit tests([#18454](https://github.com/opensearch-project/OpenSearch/pull/18454))
 - Pass index settings to system ingest processor factories. ([#18708](https://github.com/opensearch-project/OpenSearch/pull/18708))
+- Add fetch phase profiling. ([#18664](https://github.com/opensearch-project/OpenSearch/pull/18664))
 - Include named queries from rescore contexts in matched_queries array ([#18697](https://github.com/opensearch-project/OpenSearch/pull/18697))
 - Add the configurable limit on rule cardinality ([#18663](https://github.com/opensearch-project/OpenSearch/pull/18663))
 - [Experimental] Start in "clusterless" mode if a clusterless ClusterPlugin is loaded ([#18479](https://github.com/opensearch-project/OpenSearch/pull/18479))

modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/130_script_fields_profile.yml

@@ -0,0 +1,69 @@
+setup:
+  - do:
+      indices.create:
+        index: test_fetch_profile
+        body:
+          settings:
+            number_of_replicas: 0
+            number_of_shards: 1
+          mappings:
+            properties:
+              text_field:
+                type: text
+                fields:
+                  keyword:
+                    type: keyword
+              numeric_field:
+                type: integer
+              date_field:
+                type: date
+              object_field:
+                type: nested
+                properties:
+                  nested_field:
+                    type: keyword
+              stored_field:
+                type: keyword
+                store: true
+
+  - do:
+      bulk:
+        refresh: true
+        index: test_fetch_profile
+        body: |
+          { "index": {} }
+          { "text_field": "Hello world", "numeric_field": 42, "date_field": "2023-01-01", "object_field": { "nested_field": "nested value" }, "stored_field": "stored value" }
+          { "index": {} }
+          { "text_field": "Another document", "numeric_field": 100, "date_field": "2023-02-01", "object_field": { "nested_field": "another nested" }, "stored_field": "another stored" }
+          { "index": {} }
+          { "text_field": "Third document with more text", "numeric_field": 200, "date_field": "2023-03-01", "object_field": { "nested_field": "third nested" }, "stored_field": "third stored" }
+
+
+---
+"Script fields phase profiling":
+  - skip:
+      features: "contains"
+
+  - do:
+      search:
+        index: test_fetch_profile
+        body:
+          profile: true
+          query:
+            match_all: {}
+          script_fields:
+            my_field:
+              script:
+                lang: painless
+                source: "doc['numeric_field'].value * 2"
+
+  - contains:
+      profile.shards.0.fetch.0.children:
+        type: "ScriptFieldsPhase"
+        description: "ScriptFieldsPhase"
+  - length: { profile.shards.0.fetch.0.children: 1 }
+
+  - is_true: profile.shards.0.fetch.0.children.0.breakdown.process
+  - match: { profile.shards.0.fetch.0.children.0.breakdown.process_count: 3 }
+  - is_true: profile.shards.0.fetch.0.children.0.breakdown.set_next_reader
+  - match: { profile.shards.0.fetch.0.children.0.breakdown.set_next_reader_count: 1 }

qa/multi-cluster-search/src/test/java/org/opensearch/search/CCSDuelIT.java

@@ -831,10 +831,12 @@ private static Map<String, Object> responseToMap(SearchResponse response) throws
         Map<String, Object> responseMap = org.opensearch.common.xcontent.XContentHelper.convertToMap(bytesReference, false, MediaTypeRegistry.JSON).v2();
         assertNotNull(responseMap.put("took", -1));
         responseMap.remove("num_reduce_phases");
-        Map<String, Object> profile = (Map<String, Object>)responseMap.get("profile");
+        Map<String, Object> profile = (Map<String, Object>) responseMap.get("profile");
         if (profile != null) {
-            List<Map<String, Object>> shards = (List <Map<String, Object>>)profile.get("shards");
+            List<Map<String, Object>> shards = (List<Map<String, Object>>) profile.get("shards");
             for (Map<String, Object> shard : shards) {
+                // Unconditionally remove the fetch profile to prevent flaky failures
+                shard.remove("fetch");
                 replaceProfileTime(shard);
             }
         }

rest-api-spec/src/main/resources/rest-api-spec/test/search.profile/10_fetch_phase.yml

@@ -0,0 +1,193 @@
+# Setup remains the same
+setup:
+  - do:
+      indices.create:
+        index: test_fetch_profile
+        body:
+          settings:
+            number_of_replicas: 0
+            number_of_shards: 1
+          mappings:
+            properties:
+              text_field:
+                type: text
+                fields:
+                  keyword:
+                    type: keyword
+              numeric_field:
+                type: integer
+              date_field:
+                type: date
+              object_field:
+                type: nested
+                properties:
+                  nested_field:
+                    type: keyword
+              stored_field:
+                type: keyword
+                store: true
+
+  - do:
+      bulk:
+        refresh: true
+        index: test_fetch_profile
+        body: |
+          { "index": {} }
+          { "text_field": "Hello world", "numeric_field": 42, "date_field": "2023-01-01", "object_field": { "nested_field": "nested value" }, "stored_field": "stored value" }
+          { "index": {} }
+          { "text_field": "Another document", "numeric_field": 100, "date_field": "2023-02-01", "object_field": { "nested_field": "another nested" }, "stored_field": "another stored" }
+          { "index": {} }
+          { "text_field": "Third document with more text", "numeric_field": 200, "date_field": "2023-03-01", "object_field": { "nested_field": "third nested" }, "stored_field": "third stored" }
+
+---
+"Combined fetch sub-phases profiling":
+  - skip:
+      version: " - 3.1.99"
+      reason: "Fetch phase profiling was introduced in 3.2.0"
+      features: "contains"
+
+  - do:
+      search:
+        index: test_fetch_profile
+        body:
+          profile: true
+          explain: true
+          version: true
+          seq_no_primary_term: true
+          track_scores: true
+          sort:
+            - numeric_field: { order: asc }
+          query:
+            bool:
+              must:
+                - match_all: {}
+              should:
+                - nested:
+                    path: "object_field"
+                    query:
+                      wildcard:
+                        "object_field.nested_field": "nested*"
+                    inner_hits: {}
+                - match:
+                    text_field:
+                      query: "document"
+                      _name: "my_query"
+          docvalue_fields:
+            - "numeric_field"
+          fields:
+            - "stored_field"
+          highlight:
+            fields:
+              "object_field.nested_field": {}
+
+  # 1. Verify basic fetch profile structure
+  - is_true: profile.shards.0.fetch.0
+  - match: { profile.shards.0.fetch.0.type: "fetch" }
+  - match: { profile.shards.0.fetch.0.description: "fetch" }
+  - is_true: profile.shards.0.fetch.0.time_in_nanos
+
+  # 2. Verify detailed breakdown of the main fetch operation
+  - is_true: profile.shards.0.fetch.0.breakdown
+  - is_true: profile.shards.0.fetch.0.breakdown.load_stored_fields
+  - match: { profile.shards.0.fetch.0.breakdown.load_stored_fields_count: 3 }
+  - is_true: profile.shards.0.fetch.0.breakdown.load_source
+  - match: { profile.shards.0.fetch.0.breakdown.load_source_count: 3 }
+  - is_true: profile.shards.0.fetch.0.breakdown.get_next_reader
+  - match: { profile.shards.0.fetch.0.breakdown.get_next_reader_count: 1}
+  - is_true: profile.shards.0.fetch.0.breakdown.build_sub_phase_processors
+  - match: { profile.shards.0.fetch.0.breakdown.build_sub_phase_processors_count: 1}
+  - is_true: profile.shards.0.fetch.0.breakdown.create_stored_fields_visitor
+  - match: { profile.shards.0.fetch.0.breakdown.create_stored_fields_visitor_count: 1}
+
+  # 3. Verify all expected fetch sub-phases are present as children
+  - length: { profile.shards.0.fetch.0.children: 9 }
+  - contains:
+      profile.shards.0.fetch.0.children:
+        type: "FetchSourcePhase"
+  - contains:
+      profile.shards.0.fetch.0.children:
+        type: "ExplainPhase"
+  - contains:
+      profile.shards.0.fetch.0.children:
+        type: "FetchDocValuesPhase"
+  - contains:
+      profile.shards.0.fetch.0.children:
+        type: "FetchFieldsPhase"
+  - contains:
+      profile.shards.0.fetch.0.children:
+        type: "FetchVersionPhase"
+  - contains:
+      profile.shards.0.fetch.0.children:
+        type: "SeqNoPrimaryTermPhase"
+  - contains:
+      profile.shards.0.fetch.0.children:
+        type: "MatchedQueriesPhase"
+  - contains:
+      profile.shards.0.fetch.0.children:
+        type: "HighlightPhase"
+  - contains:
+      profile.shards.0.fetch.0.children:
+        type: "FetchScorePhase"
+
+---
+"No source or empty fetch profiling":
+  - skip:
+      version: " - 3.1.99"
+      reason: "Fetch phase profiling was introduced in 3.2.0"
+
+  # Case 1: Test with _source: false, which removes FetchSourcePhase
+  - do:
+      search:
+        index: test_fetch_profile
+        body:
+          profile: true
+          query:
+            match_all: {}
+          _source: false
+          docvalue_fields:
+            - "numeric_field"
+
+  - is_true: profile.shards.0.fetch.0
+  - length: { profile.shards.0.fetch.0.children: 1 }
+  - match: { profile.shards.0.fetch.0.children.0.type: "FetchDocValuesPhase" }
+  - match: { profile.shards.0.fetch.0.children.0.breakdown.process_count: 3 }
+
+  # Case 2: Test with size: 0, which results in an empty fetch profile
+  - do:
+      search:
+        index: test_fetch_profile
+        body:
+          profile: true
+          size: 0
+          aggs:
+            group_by_nested:
+              terms:
+                field: "object_field.nested_field"
+
+  - match: { profile.shards.0.fetch: [ ] }
+
+---
+"Top-hits aggregation profiling":
+  - skip:
+      version: " - 3.1.99"
+      reason: "Fetch phase profiling was introduced in 3.2.0"
+
+  - do:
+      search:
+        index: test_fetch_profile
+        body:
+          profile: true
+          query:
+            match:
+              text_field: "document"
+          aggs:
+            top_hits_agg:
+              top_hits:
+                size: 1
+
+  # Verify that the profile contains a single fetch operation for the query
+  - length: { profile.shards.0.fetch: 1 }
+  - match: { profile.shards.0.fetch.0.type: "fetch" }
+  - match: { profile.shards.0.fetch.0.description: "fetch" }
+  - length: { profile.shards.0.fetch.0.children: 1 }
+  - match: { profile.shards.0.fetch.0.children.0.type: "FetchSourcePhase" }