Fix test DeletePitMultiNodeIT.testDeleteWhileSearch (opensearch-proje…

…ct#9482)

Signed-off-by: panguixin <panguixin@bytedance.com>

server/src/internalClusterTest/java/org/opensearch/search/pit/DeletePitMultiNodeIT.java

@@ -8,6 +8,7 @@
 
 package org.opensearch.search.pit;
 
+import org.opensearch.ExceptionsHelper;
 import org.opensearch.action.LatchedActionListener;
 import org.opensearch.action.admin.indices.stats.IndicesStatsRequest;
 import org.opensearch.action.admin.indices.stats.IndicesStatsResponse;
@@ -18,10 +19,14 @@
 import org.opensearch.action.search.DeletePitInfo;
 import org.opensearch.action.search.DeletePitRequest;
 import org.opensearch.action.search.DeletePitResponse;
+import org.opensearch.action.search.SearchPhaseExecutionException;
+import org.opensearch.action.search.SearchResponse;
+import org.opensearch.action.search.ShardSearchFailure;
 import org.opensearch.common.action.ActionFuture;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.action.ActionListener;
+import org.opensearch.search.SearchContextMissingException;
 import org.opensearch.search.builder.PointInTimeBuilder;
 import org.opensearch.test.InternalTestCluster;
 import org.opensearch.test.OpenSearchIntegTestCase;
@@ -263,18 +268,23 @@ public void testDeleteWhileSearch() throws Exception {
                 try {
                     latch.await();
                     for (int j = 0; j < 30; j++) {
-                        client().prepareSearch()
+                        SearchResponse searchResponse = client().prepareSearch()
                             .setSize(2)
                             .setPointInTime(new PointInTimeBuilder(pitResponse.getId()).setKeepAlive(TimeValue.timeValueDays(1)))
                             .execute()
                             .get();
+                        if (searchResponse.getFailedShards() != 0) {
+                            verifySearchContextMissingException(searchResponse.getShardFailures());
+                        }
                     }
                 } catch (Exception e) {
                     /**
                      * assert for exception once delete pit goes through. throw error in case of any exeption before that.
                      */
                     if (deleted.get() == true) {
-                        if (!e.getMessage().contains("all shards failed")) throw new AssertionError(e);
+                        Throwable t = ExceptionsHelper.unwrapCause(e.getCause());
+                        assertTrue(e.toString(), t instanceof SearchPhaseExecutionException);
+                        verifySearchContextMissingException(((SearchPhaseExecutionException) t).shardFailures());
                         return;
                     }
                     throw new AssertionError(e);
@@ -283,9 +293,9 @@ public void testDeleteWhileSearch() throws Exception {
             threads[i].setName("opensearch[node_s_0][search]");
             threads[i].start();
         }
+        deleted.set(true);
         ActionFuture<DeletePitResponse> execute = client().execute(DeletePitAction.INSTANCE, deletePITRequest);
         DeletePitResponse deletePITResponse = execute.get();
-        deleted.set(true);
         for (DeletePitInfo deletePitInfo : deletePITResponse.getDeletePitResults()) {
             assertTrue(pitIds.contains(deletePitInfo.getPitId()));
             assertTrue(deletePitInfo.isSuccessful());
@@ -296,6 +306,13 @@ public void testDeleteWhileSearch() throws Exception {
         }
     }
 
+    private void verifySearchContextMissingException(ShardSearchFailure[] failures) {
+        for (ShardSearchFailure failure : failures) {
+            Throwable cause = ExceptionsHelper.unwrapCause(failure.getCause());
+            assertTrue(failure.toString(), cause instanceof SearchContextMissingException);
+        }
+    }
+
     public void testtConcurrentDeletes() throws InterruptedException, ExecutionException {
         CreatePitResponse pitResponse = createPitOnIndex("index");
         ensureGreen();