Test doc-value-based searches on older indices

test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java

@@ -447,7 +447,7 @@ public void test() throws IOException {
             inFipsJvm() && testCandidate.getTestSection().getSkipSection().getFeatures().contains("fips_140")
         );
 
-        if (testCandidate.getSetupSection().isEmpty() == false) {
+        if (skipSetupSections() == false && testCandidate.getSetupSection().isEmpty() == false) {
             logger.debug("start setup test [{}]", testCandidate.getTestPath());
             for (ExecutableSection executableSection : testCandidate.getSetupSection().getExecutableSections()) {
                 executeSection(executableSection);
@@ -470,6 +470,10 @@ public void test() throws IOException {
         }
     }
 
+    protected boolean skipSetupSections() {
+        return false;
+    }
+
     /**
      * Execute an {@link ExecutableSection}, careful to log its place of origin on failure.
      */

x-pack/qa/repository-old-versions/build.gradle

@@ -18,6 +18,7 @@ import org.gradle.api.internal.artifacts.ArtifactAttributes
 apply plugin: 'elasticsearch.jdk-download'
 apply plugin: 'elasticsearch.internal-testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
+apply plugin: 'elasticsearch.rest-resources'
 
 configurations {
   oldesFixture
@@ -37,6 +38,15 @@ jdks {
   }
 }
 
+restResources {
+  restApi {
+    include '_common', 'search'
+  }
+  restTests {
+    includeCore 'search/390_doc_values_search.yml'
+  }
+}
+
 if (Os.isFamily(Os.FAMILY_WINDOWS)) {
   logger.warn("Disabling repository-old-versions tests because we can't get the pid file on windows")
   tasks.named("testingConventions").configure { enabled = false }
@@ -91,15 +101,21 @@ if (Os.isFamily(Os.FAMILY_WINDOWS)) {
       if (Architecture.current() == Architecture.AARCH64) {
         env 'ES_JAVA_OPTS', '-Xss512k'
       }
+      def dataPath = "${baseDir}/data"
       args 'oldes.OldElasticsearch',
         baseDir,
         "${ -> config.getSingleFile().toPath()}",
         false,
-        "path.repo: ${repoLocation}"
+        "path.repo: ${repoLocation}",
+        "path.data: ${dataPath}"
       if (version.onOrAfter('6.8.0') && Architecture.current() == Architecture.AARCH64) {
         // We need to explicitly disable ML when running old ES versions on ARM
         args 'xpack.ml.enabled: false'
       }
+      doFirst {
+        delete(dataPath)
+        mkdir(dataPath)
+      }
       maxWaitInSeconds 60
       waitCondition = { fixture, ant ->
         // the fixture writes the ports file when Elasticsearch's HTTP service

x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/DocValueOnlyFieldsIT.java

@@ -0,0 +1,222 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.oldrepos;
+
+import com.carrotsearch.randomizedtesting.RandomizedTest;
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import org.apache.http.HttpHost;
+import org.elasticsearch.Build;
+import org.elasticsearch.Version;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.core.Booleans;
+import org.elasticsearch.core.PathUtils;
+import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
+import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentFactory;
+import org.junit.Before;
+
+import java.io.IOException;
+
+/**
+ * Tests doc-value-based searches against indices imported from clusters older than N-1.
+ * We reuse the YAML tests in search/390_doc_values_search.yml but have to do the setup
+ * manually here as the setup is done on the old cluster for which we have to use the
+ * low-level REST client instead of the YAML set up that only knows how to talk to
+ * newer ES versions.
+ *
+ * We mimic the setup in search/390_doc_values_search.yml here, but adapt it to work
+ * against older version clusters.
+ */
+public class DocValueOnlyFieldsIT extends ESClientYamlSuiteTestCase {
+
+    final Version oldVersion = Version.fromString(System.getProperty("tests.es.version"));
+    static boolean setupDone;
+
+    public DocValueOnlyFieldsIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws Exception {
+        return ESClientYamlSuiteTestCase.createParameters();
+    }
+
+    @Override
+    protected boolean preserveClusterUponCompletion() {
+        return true;
+    }
+
+    @Override
+    protected Settings restClientSettings() {
+        String token = basicAuthHeaderValue("admin", new SecureString("admin-password".toCharArray()));
+        return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build();
+    }
+
+    @Override
+    public void test() throws IOException {
+        assumeTrue("feature currently only enabled in snapshot builds", Build.CURRENT.isSnapshot());
+        super.test();
+    }
+
+    @Override
+    protected boolean skipSetupSections() {
+        // setup in the YAML file is replaced by the method below
+        return true;
+    }
+
+    @Before
+    public void setupIndex() throws IOException {
+        assumeTrue("feature currently only enabled in snapshot builds", Build.CURRENT.isSnapshot());
+
+        final boolean afterRestart = Booleans.parseBoolean(System.getProperty("tests.after_restart"));
+        if (afterRestart) {
+            return;
+        }
+
+        // The following is bit of a hack. While we wish we could make this an @BeforeClass, it does not work because the client() is only
+        // initialized later, so we do it when running the first test
+        if (setupDone) {
+            return;
+        }
+
+        setupDone = true;
+
+        String repoLocation = PathUtils.get(System.getProperty("tests.repo.location"))
+            .resolve(RandomizedTest.getContext().getTargetClass().getName())
+            .toString();
+
+        String indexName = "test";
+        String repoName = "doc_values_repo";
+        String snapshotName = "snap";
+        String[] basicTypes = new String[] {
+            "byte",
+            "double",
+            "float",
+            "half_float",
+            "integer",
+            "long",
+            "short",
+            "boolean",
+            "keyword",
+            "ip",
+            "geo_point" }; // date is manually added as it need further configuration
+
+        int oldEsPort = Integer.parseInt(System.getProperty("tests.es.port"));
+        try (RestClient oldEs = RestClient.builder(new HttpHost("127.0.0.1", oldEsPort)).build()) {
+            Request createIndex = new Request("PUT", "/" + indexName);
+            int numberOfShards = randomIntBetween(1, 3);
+
+            boolean multiTypes = oldVersion.before(Version.V_7_0_0);
+
+            XContentBuilder settingsBuilder = XContentFactory.jsonBuilder()
+                .startObject()
+                .startObject("settings")
+                .field("index.number_of_shards", numberOfShards)
+                .endObject()
+                .startObject("mappings");
+            if (multiTypes) {
+                settingsBuilder.startObject("doc");
+            }
+            settingsBuilder.field("dynamic", false).startObject("properties");
+            for (String type : basicTypes) {
+                settingsBuilder.startObject(type).field("type", type).endObject();
+            }
+            settingsBuilder.startObject("date").field("type", "date").field("format", "yyyy/MM/dd").endObject();
+            if (multiTypes) {
+                settingsBuilder.endObject();
+            }
+            settingsBuilder.endObject().endObject().endObject();
+
+            createIndex.setJsonEntity(Strings.toString(settingsBuilder));
+            assertOK(oldEs.performRequest(createIndex));
+
+            Request doc1 = new Request("PUT", "/" + indexName + "/" + "doc" + "/" + "1");
+            doc1.addParameter("refresh", "true");
+            XContentBuilder bodyDoc1 = XContentFactory.jsonBuilder()
+                .startObject()
+                .field("byte", 1)
+                .field("double", 1.0)
+                .field("float", 1.0)
+                .field("half_float", 1.0)
+                .field("integer", 1)
+                .field("long", 1)
+                .field("short", 1)
+                .field("date", "2017/01/01")
+                .field("keyword", "key1")
+                .field("boolean", false)
+                .field("ip", "192.168.0.1")
+                .array("geo_point", 13.5, 34.89)
+                .endObject();
+            doc1.setJsonEntity(Strings.toString(bodyDoc1));
+            assertOK(oldEs.performRequest(doc1));
+
+            Request doc2 = new Request("PUT", "/" + indexName + "/" + "doc" + "/" + "2");
+            doc2.addParameter("refresh", "true");
+            XContentBuilder bodyDoc2 = XContentFactory.jsonBuilder()
+                .startObject()
+                .field("byte", 2)
+                .field("double", 2.0)
+                .field("float", 2.0)
+                .field("half_float", 2.0)
+                .field("integer", 2)
+                .field("long", 2)
+                .field("short", 2)
+                .field("date", "2017/01/02")
+                .field("keyword", "key2")
+                .field("boolean", true)
+                .field("ip", "192.168.0.2")
+                .array("geo_point", -63.24, 31.0)
+                .endObject();
+            doc2.setJsonEntity(Strings.toString(bodyDoc2));
+            assertOK(oldEs.performRequest(doc2));
+
+            // register repo on old ES and take snapshot
+            Request createRepoRequest = new Request("PUT", "/_snapshot/" + repoName);
+            createRepoRequest.setJsonEntity("""
+                {"type":"fs","settings":{"location":"%s"}}
+                """.formatted(repoLocation));
+            assertOK(oldEs.performRequest(createRepoRequest));
+
+            Request createSnapshotRequest = new Request("PUT", "/_snapshot/" + repoName + "/" + snapshotName);
+            createSnapshotRequest.addParameter("wait_for_completion", "true");
+            createSnapshotRequest.setJsonEntity("{\"indices\":\"" + indexName + "\"}");
+            assertOK(oldEs.performRequest(createSnapshotRequest));
+        }
+
+        // register repo on new ES and restore snapshot
+        Request createRepoRequest2 = new Request("PUT", "/_snapshot/" + repoName);
+        createRepoRequest2.setJsonEntity("""
+            {"type":"fs","settings":{"location":"%s","allow_bwc_indices":true}}
+            """.formatted(repoLocation));
+        assertOK(client().performRequest(createRepoRequest2));
+
+        final Request createRestoreRequest = new Request("POST", "/_snapshot/" + repoName + "/" + snapshotName + "/_restore");
+        createRestoreRequest.addParameter("wait_for_completion", "true");
+        createRestoreRequest.setJsonEntity("{\"indices\":\"" + indexName + "\"}");
+        assertOK(client().performRequest(createRestoreRequest));
+
+        // add mappings (they will be auto-converted later)
+        Request putMappingsRequest = new Request("PUT", "/" + indexName + "/_mappings");
+        XContentBuilder mappingsBuilder = XContentFactory.jsonBuilder().startObject().startObject("properties");
+        for (String type : basicTypes) {
+            mappingsBuilder.startObject(type).field("type", type).field("index", false).endObject();
+        }
+        mappingsBuilder.startObject("date").field("type", "date").field("index", false).field("format", "yyyy/MM/dd").endObject();
+        mappingsBuilder.endObject().endObject();
+        putMappingsRequest.setJsonEntity(Strings.toString(mappingsBuilder));
+        assertOK(client().performRequest(putMappingsRequest));
+    }
+}