chore(SemanticTextHighlighter): improve bwc tests

x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/90_semantic_text_highlighter.yml

@@ -745,81 +745,52 @@ setup:
   - match: { hits.hits.0.highlight.body.1: "You Know, for Search!" }
 
 ---
-"Highlighting with type:bbq_disk index options as well as knn with similarity":
+"Highlighting with knn with similarity":
   - requires:
-      cluster_features: "semantic_text.highlighter.bbq_and_similarity_support"
-      reason: semantic highlighter fix for disk_bbq index options and knn with similarity
-
-  - do:
-      indices.create:
-        index: test-dense-index-bbq_disk
-        body:
-          settings:
-            index.mapping.semantic_text.use_legacy_format: false
-          mappings:
-            properties:
-              bbq_disk_body:
-                type: semantic_text
-                inference_id: dense-inference-id
-                index_options:
-                  dense_vector:
-                    type: bbq_disk
-              body:
-                type: semantic_text
-                inference_id: dense-inference-id
+      cluster_features: "semantic_text.highlighter.vector_similarity_support"
+      reason: semantic highlighter fix for knn with similarity
 
   - do:
       index:
-        index: test-dense-index-bbq_disk
+        index: test-dense-index
         id: doc_1
         body:
-          bbq_disk_body: [ "ElasticSearch is an open source, distributed, RESTful, search engine which is built on top of Lucene internally and enjoys all the features it provides.", "You Know, for Search!", "For a moment, nothing happened. Then, after a second or so, nothing continued to happen." ]
           body: [ "ElasticSearch is an open source, distributed, RESTful, search engine which is built on top of Lucene internally and enjoys all the features it provides.", "You Know, for Search!", "For a moment, nothing happened. Then, after a second or so, nothing continued to happen." ]
   - do:
       index:
-        index: test-dense-index-bbq_disk
+        index: test-dense-index
         id: doc_2
         body:
-          bbq_disk_body: [ "Nothing travels faster than the speed of light with the possible exception of bad news, which obeys its own special laws."]
           body: [ "Nothing travels faster than the speed of light with the possible exception of bad news, which obeys its own special laws."]
         refresh: true
 
   - do:
       search:
-        index: test-dense-index-bbq_disk
+        index: test-dense-index
         body:
           query:
             match_all: { }
           highlight:
             fields:
-              bbq_disk_body:
-                type: "semantic"
-                number_of_fragments: 1
               body:
                 type: "semantic"
                 number_of_fragments: 1
 
   - match: { hits.total.value: 2 }
 
   - match: { hits.hits.0._id: "doc_1" }
-  - length: { hits.hits.0.highlight: 2 }
-  - length: { hits.hits.0.highlight.bbq_disk_body: 1 }
-  - match: { hits.hits.0.highlight.bbq_disk_body.0: "ElasticSearch is an open source, distributed, RESTful, search engine which is built on top of Lucene internally and enjoys all the features it provides." }
+  - length: { hits.hits.0.highlight: 1 }
   - length: { hits.hits.0.highlight.body: 1 }
   - match: { hits.hits.0.highlight.body.0: "ElasticSearch is an open source, distributed, RESTful, search engine which is built on top of Lucene internally and enjoys all the features it provides." }
 
   - match: { hits.hits.1._id: "doc_2" }
-  - length: { hits.hits.1.highlight: 2 }
-  - length: { hits.hits.1.highlight.bbq_disk_body: 1 }
-  - match: { hits.hits.1.highlight.bbq_disk_body.0: "Nothing travels faster than the speed of light with the possible exception of bad news, which obeys its own special laws." }
+  - length: { hits.hits.1.highlight: 1 }
   - length: { hits.hits.1.highlight.body: 1 }
   - match: { hits.hits.1.highlight.body.0: "Nothing travels faster than the speed of light with the possible exception of bad news, which obeys its own special laws." }
 
-
-
   - do:
       search:
-        index: test-dense-index-bbq_disk
+        index: test-dense-index
         body:
           query:
             knn:
@@ -843,6 +814,65 @@ setup:
   - match: { hits.hits.0.highlight.body.1: "You Know, for Search!" }
   - match: { hits.hits.0.highlight.body.2: "For a moment, nothing happened. Then, after a second or so, nothing continued to happen."}
 
+---
+"Highlighting with type:bbq_disk index options":
+  - requires:
+      cluster_features: "semantic_text.highlighter.bbq_and_similarity_support"
+      reason: semantic highlighter fix for disk_bbq index options and knn with similarity
+
+  - do:
+      indices.create:
+        index: test-dense-index-bbq_disk
+        body:
+          settings:
+            index.mapping.semantic_text.use_legacy_format: false
+          mappings:
+            properties:
+              bbq_disk_body:
+                type: semantic_text
+                inference_id: dense-inference-id-compatible-with-bbq
+                index_options:
+                  dense_vector:
+                    type: bbq_disk
+
+  - do:
+      index:
+        index: test-dense-index-bbq_disk
+        id: doc_1
+        body:
+          bbq_disk_body: [ "ElasticSearch is an open source, distributed, RESTful, search engine which is built on top of Lucene internally and enjoys all the features it provides.", "You Know, for Search!", "For a moment, nothing happened. Then, after a second or so, nothing continued to happen." ]
+  - do:
+      index:
+        index: test-dense-index-bbq_disk
+        id: doc_2
+        body:
+          bbq_disk_body: [ "Nothing travels faster than the speed of light with the possible exception of bad news, which obeys its own special laws."]
+        refresh: true
+
+  - do:
+      search:
+        index: test-dense-index-bbq_disk
+        body:
+          query:
+            match_all: { }
+          highlight:
+            fields:
+              bbq_disk_body:
+                type: "semantic"
+                number_of_fragments: 1
+
+  - match: { hits.total.value: 2 }
+
+  - match: { hits.hits.0._id: "doc_1" }
+  - length: { hits.hits.0.highlight: 1 }
+  - length: { hits.hits.0.highlight.bbq_disk_body: 1 }
+  - match: { hits.hits.0.highlight.bbq_disk_body.0: "ElasticSearch is an open source, distributed, RESTful, search engine which is built on top of Lucene internally and enjoys all the features it provides." }
+
+  - match: { hits.hits.1._id: "doc_2" }
+  - length: { hits.hits.1.highlight: 1 }
+  - length: { hits.hits.1.highlight.bbq_disk_body: 1 }
+  - match: { hits.hits.1.highlight.bbq_disk_body.0: "Nothing travels faster than the speed of light with the possible exception of bad news, which obeys its own special laws." }
+
   - do:
       search:
         index: test-dense-index-bbq_disk
@@ -855,18 +885,19 @@ setup:
                   model_text: "What is Elasticsearch?"
               k: 10
               num_candidates: 10
-              similarity: 0.9977
+              similarity: 0.9975
           highlight:
             fields:
-              body:
+              bbq_disk_body:
                 type: "semantic"
-                number_of_fragments: 2
+                number_of_fragments: 3
 
   - match: { hits.total.value: 1 }
   - match: { hits.hits.0._id: "doc_1" }
-  - length: { hits.hits.0.highlight.body: 2 }
-  - match: { hits.hits.0.highlight.body.0: "ElasticSearch is an open source, distributed, RESTful, search engine which is built on top of Lucene internally and enjoys all the features it provides." }
-  - match: { hits.hits.0.highlight.body.1: "For a moment, nothing happened. Then, after a second or so, nothing continued to happen."}
+  - length: { hits.hits.0.highlight.bbq_disk_body: 3 }
+  - match: { hits.hits.0.highlight.bbq_disk_body.0: "ElasticSearch is an open source, distributed, RESTful, search engine which is built on top of Lucene internally and enjoys all the features it provides." }
+  - match: { hits.hits.0.highlight.bbq_disk_body.1: "You Know, for Search!" }
+  - match: { hits.hits.0.highlight.bbq_disk_body.2: "For a moment, nothing happened. Then, after a second or so, nothing continued to happen."}
 
   - do:
       search:

x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/90_semantic_text_highlighter_bwc.yml

@@ -650,81 +650,52 @@ setup:
   - match: { hits.hits.0.highlight.bbq_hnsw_field.0: "ElasticSearch is an open source, distributed, RESTful, search engine which is built on top of Lucene internally and enjoys all the features it provides." }
 
 ---
-"Highlighting with type:bbq_disk index options as well as knn with similarity":
+"Highlighting with knn with similarity":
   - requires:
-      cluster_features: "semantic_text.highlighter.bbq_and_similarity_support"
-      reason: semantic highlighter fix for disk_bbq index options and knn with similarity
-
-  - do:
-      indices.create:
-        index: test-dense-index-bbq_disk
-        body:
-          settings:
-            index.mapping.semantic_text.use_legacy_format: true
-          mappings:
-            properties:
-              bbq_disk_body:
-                type: semantic_text
-                inference_id: dense-inference-id
-                index_options:
-                  dense_vector:
-                    type: bbq_disk
-              body:
-                type: semantic_text
-                inference_id: dense-inference-id
+      cluster_features: "semantic_text.highlighter.vector_similarity_support"
+      reason: semantic highlighter fix for knn with similarity
 
   - do:
       index:
-        index: test-dense-index-bbq_disk
+        index: test-dense-index
         id: doc_1
         body:
-          bbq_disk_body: [ "ElasticSearch is an open source, distributed, RESTful, search engine which is built on top of Lucene internally and enjoys all the features it provides.", "You Know, for Search!", "For a moment, nothing happened. Then, after a second or so, nothing continued to happen." ]
           body: [ "ElasticSearch is an open source, distributed, RESTful, search engine which is built on top of Lucene internally and enjoys all the features it provides.", "You Know, for Search!", "For a moment, nothing happened. Then, after a second or so, nothing continued to happen." ]
   - do:
       index:
-        index: test-dense-index-bbq_disk
+        index: test-dense-index
         id: doc_2
         body:
-          bbq_disk_body: [ "Nothing travels faster than the speed of light with the possible exception of bad news, which obeys its own special laws."]
           body: [ "Nothing travels faster than the speed of light with the possible exception of bad news, which obeys its own special laws."]
         refresh: true
 
   - do:
       search:
-        index: test-dense-index-bbq_disk
+        index: test-dense-index
         body:
           query:
             match_all: { }
           highlight:
             fields:
-              bbq_disk_body:
-                type: "semantic"
-                number_of_fragments: 1
               body:
                 type: "semantic"
                 number_of_fragments: 1
 
   - match: { hits.total.value: 2 }
 
   - match: { hits.hits.0._id: "doc_1" }
-  - length: { hits.hits.0.highlight: 2 }
-  - length: { hits.hits.0.highlight.bbq_disk_body: 1 }
-  - match: { hits.hits.0.highlight.bbq_disk_body.0: "ElasticSearch is an open source, distributed, RESTful, search engine which is built on top of Lucene internally and enjoys all the features it provides." }
+  - length: { hits.hits.0.highlight: 1 }
   - length: { hits.hits.0.highlight.body: 1 }
   - match: { hits.hits.0.highlight.body.0: "ElasticSearch is an open source, distributed, RESTful, search engine which is built on top of Lucene internally and enjoys all the features it provides." }
 
   - match: { hits.hits.1._id: "doc_2" }
-  - length: { hits.hits.1.highlight: 2 }
-  - length: { hits.hits.1.highlight.bbq_disk_body: 1 }
-  - match: { hits.hits.1.highlight.bbq_disk_body.0: "Nothing travels faster than the speed of light with the possible exception of bad news, which obeys its own special laws." }
+  - length: { hits.hits.1.highlight: 1 }
   - length: { hits.hits.1.highlight.body: 1 }
   - match: { hits.hits.1.highlight.body.0: "Nothing travels faster than the speed of light with the possible exception of bad news, which obeys its own special laws." }
 
-
-
   - do:
       search:
-        index: test-dense-index-bbq_disk
+        index: test-dense-index
         body:
           query:
             knn:
@@ -748,6 +719,65 @@ setup:
   - match: { hits.hits.0.highlight.body.1: "You Know, for Search!" }
   - match: { hits.hits.0.highlight.body.2: "For a moment, nothing happened. Then, after a second or so, nothing continued to happen."}
 
+---
+"Highlighting with type:bbq_disk index options":
+  - requires:
+      cluster_features: "semantic_text.highlighter.bbq_and_similarity_support"
+      reason: semantic highlighter fix for disk_bbq index options and knn with similarity
+
+  - do:
+      indices.create:
+        index: test-dense-index-bbq_disk
+        body:
+          settings:
+            index.mapping.semantic_text.use_legacy_format: true
+          mappings:
+            properties:
+              bbq_disk_body:
+                type: semantic_text
+                inference_id: dense-inference-id-compatible-with-bbq
+                index_options:
+                  dense_vector:
+                    type: bbq_disk
+
+  - do:
+      index:
+        index: test-dense-index-bbq_disk
+        id: doc_1
+        body:
+          bbq_disk_body: [ "ElasticSearch is an open source, distributed, RESTful, search engine which is built on top of Lucene internally and enjoys all the features it provides.", "You Know, for Search!", "For a moment, nothing happened. Then, after a second or so, nothing continued to happen." ]
+  - do:
+      index:
+        index: test-dense-index-bbq_disk
+        id: doc_2
+        body:
+          bbq_disk_body: [ "Nothing travels faster than the speed of light with the possible exception of bad news, which obeys its own special laws."]
+        refresh: true
+
+  - do:
+      search:
+        index: test-dense-index-bbq_disk
+        body:
+          query:
+            match_all: { }
+          highlight:
+            fields:
+              bbq_disk_body:
+                type: "semantic"
+                number_of_fragments: 1
+
+  - match: { hits.total.value: 2 }
+
+  - match: { hits.hits.0._id: "doc_1" }
+  - length: { hits.hits.0.highlight: 1 }
+  - length: { hits.hits.0.highlight.bbq_disk_body: 1 }
+  - match: { hits.hits.0.highlight.bbq_disk_body.0: "ElasticSearch is an open source, distributed, RESTful, search engine which is built on top of Lucene internally and enjoys all the features it provides." }
+
+  - match: { hits.hits.1._id: "doc_2" }
+  - length: { hits.hits.1.highlight: 1 }
+  - length: { hits.hits.1.highlight.bbq_disk_body: 1 }
+  - match: { hits.hits.1.highlight.bbq_disk_body.0: "Nothing travels faster than the speed of light with the possible exception of bad news, which obeys its own special laws." }
+
   - do:
       search:
         index: test-dense-index-bbq_disk
@@ -760,18 +790,19 @@ setup:
                   model_text: "What is Elasticsearch?"
               k: 10
               num_candidates: 10
-              similarity: 0.9977
+              similarity: 0.9975
           highlight:
             fields:
-              body:
+              bbq_disk_body:
                 type: "semantic"
-                number_of_fragments: 2
+                number_of_fragments: 3
 
   - match: { hits.total.value: 1 }
   - match: { hits.hits.0._id: "doc_1" }
-  - length: { hits.hits.0.highlight.body: 2 }
-  - match: { hits.hits.0.highlight.body.0: "ElasticSearch is an open source, distributed, RESTful, search engine which is built on top of Lucene internally and enjoys all the features it provides." }
-  - match: { hits.hits.0.highlight.body.1: "For a moment, nothing happened. Then, after a second or so, nothing continued to happen."}
+  - length: { hits.hits.0.highlight.bbq_disk_body: 3 }
+  - match: { hits.hits.0.highlight.bbq_disk_body.0: "ElasticSearch is an open source, distributed, RESTful, search engine which is built on top of Lucene internally and enjoys all the features it provides." }
+  - match: { hits.hits.0.highlight.bbq_disk_body.1: "You Know, for Search!" }
+  - match: { hits.hits.0.highlight.bbq_disk_body.2: "For a moment, nothing happened. Then, after a second or so, nothing continued to happen."}
 
   - do:
       search: