Fix test reproducability in AbstractBuilderTestCase setup (#32403)

Currently AbstractBuilderTestCase generates certain random values in its
`beforeTest()` method annotated with @before only the first time that a test
method in the suite is run while initializing the serviceHolder that we use for
the rest of the test. This changes the values of subsequent random values
and has the effect that when running single methods from a test suite with
"-Dtests.method=*", the random values it sees are different from when the same
test method is run as part of the whole test suite. This makes it hard to use
the reproduction lines logged on failure.

This change runs the inialization of the serviceHolder and the randomization
connected to it using the test runners master seed, so reproduction by running
just one method is possible again.

Closes #32400

Author: Christoph Büscher

modules/parent-join/src/test/java/org/elasticsearch/join/query/HasChildQueryBuilderTests.java

@@ -88,9 +88,9 @@ protected Collection<Class<? extends Plugin>> getPlugins() {
     }
 
     @Override
-    protected Settings indexSettings() {
+    protected Settings createTestIndexSettings() {
         return Settings.builder()
-            .put(super.indexSettings())
+            .put(super.createTestIndexSettings())
             .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
             .build();
     }

modules/parent-join/src/test/java/org/elasticsearch/join/query/HasParentQueryBuilderTests.java

@@ -73,9 +73,9 @@ protected Collection<Class<? extends Plugin>> getPlugins() {
     }
 
     @Override
-    protected Settings indexSettings() {
+    protected Settings createTestIndexSettings() {
         return Settings.builder()
-            .put(super.indexSettings())
+            .put(super.createTestIndexSettings())
             .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
             .build();
     }

modules/parent-join/src/test/java/org/elasticsearch/join/query/LegacyHasChildQueryBuilderTests.java

@@ -20,6 +20,7 @@
 package org.elasticsearch.join.query;
 
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
+
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.ConstantScoreQuery;
@@ -111,9 +112,9 @@ protected void initializeAdditionalMappings(MapperService mapperService) throws
     }
 
     @Override
-    protected Settings indexSettings() {
+    protected Settings createTestIndexSettings() {
         return Settings.builder()
-            .put(super.indexSettings())
+            .put(super.createTestIndexSettings())
             .put("index.version.created", Version.V_5_6_0) // multi type
             .build();
     }

modules/parent-join/src/test/java/org/elasticsearch/join/query/LegacyHasParentQueryBuilderTests.java

@@ -70,9 +70,9 @@ protected Collection<Class<? extends Plugin>> getPlugins() {
     }
 
     @Override
-    protected Settings indexSettings() {
+    protected Settings createTestIndexSettings() {
         return Settings.builder()
-            .put(super.indexSettings())
+            .put(super.createTestIndexSettings())
             .put("index.version.created", Version.V_5_6_0) // legacy needs multi types
             .build();
     }

modules/parent-join/src/test/java/org/elasticsearch/join/query/ParentIdQueryBuilderTests.java

@@ -62,9 +62,9 @@ protected Collection<Class<? extends Plugin>> getPlugins() {
     }
 
     @Override
-    protected Settings indexSettings() {
+    protected Settings createTestIndexSettings() {
         return Settings.builder()
-            .put(super.indexSettings())
+            .put(super.createTestIndexSettings())
             .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
             .build();
     }

server/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java

@@ -234,7 +234,7 @@ public void testFieldsCannotBeSetToNull() {
 
     public void testDefaultFieldParsing() throws IOException {
         assumeTrue("5.x behaves differently, so skip on non-6.x indices",
-                indexVersionCreated.onOrAfter(Version.V_6_0_0_alpha1));
+                indexSettings().getIndexVersionCreated().onOrAfter(Version.V_6_0_0_alpha1));
 
         String query = randomAlphaOfLengthBetween(1, 10).toLowerCase(Locale.ROOT);
         String contentString = "{\n" +

test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java

@@ -19,6 +19,9 @@
 
 package org.elasticsearch.test;
 
+import com.carrotsearch.randomizedtesting.RandomizedTest;
+import com.carrotsearch.randomizedtesting.SeedUtils;
+
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.util.Accountable;
 import org.elasticsearch.Version;
@@ -86,6 +89,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutionException;
 import java.util.function.Function;
 import java.util.stream.Stream;
@@ -126,14 +130,13 @@ public abstract class AbstractBuilderTestCase extends ESTestCase {
         ALIAS_TO_CONCRETE_FIELD_NAME.put(GEO_POINT_ALIAS_FIELD_NAME, GEO_POINT_FIELD_NAME);
     }
 
-    protected static Version indexVersionCreated;
-
     private static ServiceHolder serviceHolder;
     private static int queryNameId = 0;
     private static Settings nodeSettings;
     private static Index index;
     private static String[] currentTypes;
     protected static String[] randomTypes;
+    private static long nowInMillis;
 
     protected static Index getIndex() {
         return index;
@@ -162,6 +165,7 @@ public static void beforeClass() {
                 .build();
 
         index = new Index(randomAlphaOfLengthBetween(1, 10), "_na_");
+        nowInMillis = randomNonNegativeLong();
 
         // Set a single type in the index
         switch (random().nextInt(3)) {
@@ -211,15 +215,19 @@ protected static String createUniqueRandomName() {
         return queryName;
     }
 
-    protected Settings indexSettings() {
+    protected Settings createTestIndexSettings() {
         // we have to prefer CURRENT since with the range of versions we support it's rather unlikely to get the current actually.
-        indexVersionCreated = randomBoolean() ? Version.CURRENT
-                : VersionUtils.randomVersionBetween(random(), null, Version.CURRENT);
+        Version indexVersionCreated = randomBoolean() ? Version.CURRENT
+                : VersionUtils.randomVersionBetween(random(), Version.V_6_0_0, Version.CURRENT);
         return Settings.builder()
             .put(IndexMetaData.SETTING_VERSION_CREATED, indexVersionCreated)
             .build();
     }
 
+    protected static IndexSettings indexSettings() {
+        return serviceHolder.idxSettings;
+    }
+
     protected static String expectedFieldName(String builderFieldName) {
         if (currentTypes.length == 0 || !isSingleType()) {
             return builderFieldName;
@@ -234,10 +242,20 @@ public static void afterClass() throws Exception {
     }
 
     @Before
-    public void beforeTest() throws IOException {
+    public void beforeTest() throws Exception {
         if (serviceHolder == null) {
-            serviceHolder = new ServiceHolder(nodeSettings, indexSettings(), getPlugins(), this);
+            // we initialize the serviceHolder and serviceHolderWithNoType just once, but need some
+            // calls to the randomness source during its setup. In order to not mix these calls with
+            // the randomness source that is later used in the test method, we use the master seed during
+            // this setup
+            long masterSeed = SeedUtils.parseSeed(RandomizedTest.getContext().getRunnerSeedAsString());
+            RandomizedTest.getContext().runWithPrivateRandomness(masterSeed, (Callable<Void>) () -> {
+                serviceHolder = new ServiceHolder(nodeSettings, createTestIndexSettings(), getPlugins(), nowInMillis,
+                        AbstractBuilderTestCase.this);
+                return null;
+            });
         }
+
         serviceHolder.clientInvocationHandler.delegate = this;
     }
 
@@ -333,11 +351,12 @@ private static class ServiceHolder implements Closeable {
         private final BitsetFilterCache bitsetFilterCache;
         private final ScriptService scriptService;
         private final Client client;
-        private final long nowInMillis = randomNonNegativeLong();
+        private final long nowInMillis;
 
-        ServiceHolder(Settings nodeSettings, Settings indexSettings,
-                      Collection<Class<? extends Plugin>> plugins, AbstractBuilderTestCase testCase) throws IOException {
+        ServiceHolder(Settings nodeSettings, Settings indexSettings, Collection<Class<? extends Plugin>> plugins, long nowInMillis,
+                AbstractBuilderTestCase testCase) throws IOException {
             Environment env = InternalSettingsPreparer.prepareEnvironment(nodeSettings, null);
+            this.nowInMillis = nowInMillis;
             PluginsService pluginsService;
             pluginsService = new PluginsService(nodeSettings, null, env.modulesFile(), env.pluginsFile(), plugins);