Allow reading _source from older snapshots

modules/lucene-5-codecs/build.gradle

@@ -0,0 +1,127 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import org.apache.tools.ant.taskdefs.condition.Os
+import org.elasticsearch.gradle.Architecture
+import org.elasticsearch.gradle.OS
+import org.elasticsearch.gradle.internal.info.BuildParams
+import org.elasticsearch.gradle.internal.test.AntFixture
+
+apply plugin: 'elasticsearch.test-with-dependencies'
+apply plugin: 'elasticsearch.jdk-download'
+apply plugin: 'elasticsearch.internal-yaml-rest-test'
+apply plugin: 'elasticsearch.java-rest-test'
+apply plugin: 'elasticsearch.yaml-rest-compat-test'
+apply plugin: 'elasticsearch.internal-cluster-test'
+
+esplugin {
+  description 'Provides codecs for reading Lucene 5 formats'
+  classname 'org.elasticsearch.oldcodecs.Lucene5Plugin'
+}
+
+project.afterEvaluate(p -> {
+  project.getTasks().withType(JavaCompile.class).configureEach(compileTask -> {
+    CompileOptions compileOptions = compileTask.getOptions()
+    List<String> compilerArgs = compileOptions.getCompilerArgs()
+    compilerArgs.remove("-Werror")
+    compilerArgs.remove("-Xlint:all,-path,-serial,-options,-deprecation,-try")
+    compilerArgs.remove("-Xdoclint:all")
+    compilerArgs.remove("-Xdoclint:-missing")
+  })
+})
+
+
+
+configurations {
+  oldesFixture
+  es2
+  es1
+}
+
+dependencies {
+  oldesFixture project(':test:fixtures:old-elasticsearch')
+  es2 'org.elasticsearch.distribution.zip:elasticsearch:2.4.5@zip'
+  es1 'org.elasticsearch:elasticsearch:1.7.6@zip'
+}
+
+jdks {
+  legacy {
+    vendor = 'adoptium'
+    version = '8u302+b08'
+    platform = OS.current().name().toLowerCase()
+    architecture = Architecture.current().name().toLowerCase()
+  }
+}
+
+String repoLocation = "${buildDir}/cluster/shared/repo"
+testClusters.matching { it.name == "javaRestTest" }.configureEach {
+  setting 'path.repo', repoLocation
+}
+
+if (Os.isFamily(Os.FAMILY_WINDOWS)) {
+  logger.warn("Disabling reindex-from-old tests because we can't get the pid file on windows")
+  tasks.named("javaRestTest").configure {
+    systemProperty "tests.fromOld", "false"
+  }
+} else if (rootProject.rootDir.toString().contains(" ")) {
+  logger.warn("Disabling reindex-from-old tests because Elasticsearch 1.7 won't start with spaces in the path")
+  tasks.named("javaRestTest").configure {
+    systemProperty "tests.fromOld", "false"
+  }
+} else {
+  /* Set up tasks to unzip and run the old versions of ES before running the
+   * integration tests. */
+  def versions = ['2', '1']
+  versions.each { version ->
+    // TODO Rene: we should be able to replace these unzip tasks with gradle artifact transforms
+    TaskProvider<Sync> unzip = tasks.register("unzipEs${version}", Sync) {
+      Configuration oldEsDependency = configurations['es' + version]
+      dependsOn oldEsDependency
+      /* Use a closure here to delay resolution of the dependency until we need
+       * it */
+      from {
+        oldEsDependency.collect { zipTree(it) }
+      }
+      into temporaryDir
+    }
+
+    TaskProvider<AntFixture> fixture = tasks.register("oldEs${version}Fixture", AntFixture) {
+      dependsOn project.configurations.oldesFixture, jdks.legacy
+      dependsOn unzip
+      executable = "${BuildParams.runtimeJavaHome}/bin/java"
+      env 'CLASSPATH', "${-> project.configurations.oldesFixture.asPath}"
+      // old versions of Elasticsearch need JAVA_HOME
+      env 'JAVA_HOME', jdks.legacy.javaHomePath
+      // If we are running on certain arm systems we need to explicitly set the stack size to overcome JDK page size bug
+      if (Architecture.current() == Architecture.AARCH64) {
+        env 'ES_JAVA_OPTS', '-Xss512k'
+      }
+      args 'oldes.OldElasticsearch',
+        baseDir,
+        unzip.get().temporaryDir,
+        version == '090',
+        "path.repo: ${repoLocation}"
+      waitCondition = { fixture, ant ->
+        // the fixture writes the ports file when Elasticsearch's HTTP service
+        // is ready, so we can just wait for the file to exist
+        return fixture.portsFile.exists()
+      }
+    }
+
+    tasks.named("javaRestTest").configure {
+      dependsOn fixture
+      systemProperty "tests.fromOld", "true"
+      systemProperty "tests.repoLocation", repoLocation
+      /* Use a closure on the string to delay evaluation until right before we
+       * run the integration tests so that we can be sure that the file is
+       * ready. */
+      nonInputProperties.systemProperty "es${version}.port", "${-> fixture.get().addressAndPort}"
+    }
+  }
+}
+

modules/lucene-5-codecs/src/javaRestTest/java/org/elasticsearch/oldcodecs/BWCCodecIT.java

@@ -0,0 +1,139 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.oldcodecs;
+
+import org.apache.http.HttpHost;
+import org.apache.http.util.EntityUtils;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.common.io.Streams;
+import org.elasticsearch.core.Booleans;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.hamcrest.Matcher;
+
+import java.io.IOException;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.not;
+
+public class BWCCodecIT extends ESRestTestCase {
+    private static final int DOCS = 5;
+
+    private void oldEsTestCase(String portPropertyName, String requestsPerSecond) throws IOException {
+        boolean enabled = Booleans.parseBoolean(System.getProperty("tests.fromOld"));
+        assumeTrue("test is disabled, probably because this is windows", enabled);
+        String repoLocation = System.getProperty("tests.repoLocation");
+        repoLocation = repoLocation + "/" + randomAlphaOfLength(10);
+
+        int oldEsPort = Integer.parseInt(System.getProperty(portPropertyName));
+        try (RestClient oldEs = RestClient.builder(new HttpHost("127.0.0.1", oldEsPort)).build()) {
+            try {
+                Request createIndex = new Request("PUT", "/test");
+                createIndex.setJsonEntity("{\"settings\":{\"number_of_shards\": 1}}");
+                oldEs.performRequest(createIndex);
+
+                for (int i = 0; i < DOCS; i++) {
+                    Request doc = new Request("PUT", "/test/doc/testdoc" + i);
+                    doc.addParameter("refresh", "true");
+                    doc.setJsonEntity("{\"test\":\"test" + i + "\", \"val\":" + i + "}");
+                    oldEs.performRequest(doc);
+                }
+
+                // register repo on old ES and take snapshot
+                Request createRepoRequest = new Request("PUT", "/_snapshot/testrepo");
+                createRepoRequest.setJsonEntity("{\"type\":\"fs\",\"settings\":{\"location\":\"" + repoLocation + "\"}}");
+                oldEs.performRequest(createRepoRequest);
+
+                Request createSnapshotRequest = new Request("PUT", "/_snapshot/testrepo/snap1");
+                createSnapshotRequest.addParameter("wait_for_completion", "true");
+                oldEs.performRequest(createSnapshotRequest);
+
+                // register repo on new ES
+                Request createReadRepoRequest = new Request("PUT", "/_snapshot/testrepo");
+                createReadRepoRequest.setJsonEntity(
+                    "{\"type\":\"lucene5\",\"settings\":{\"delegate_type\":\"fs\",\"location\":\"" + repoLocation +
+                        "\",\"readonly\":true}}");
+                client().performRequest(createReadRepoRequest);
+
+                // list snapshots on new ES
+                Request listSnapshotsRequest = new Request("GET", "/_snapshot/testrepo/snap1");
+                listSnapshotsRequest.addParameter("error_trace", "true");
+                Response listSnapshotsResponse = client().performRequest(listSnapshotsRequest);
+                logger.info(Streams.readFully(listSnapshotsResponse.getEntity().getContent()).utf8ToString());
+                assertEquals(200, listSnapshotsResponse.getStatusLine().getStatusCode());
+
+                // list advanced snapshot info on new ES
+                Request listSnapshotStatusRequest = new Request("GET", "/_snapshot/testrepo/snap1/_status");
+                listSnapshotStatusRequest.addParameter("error_trace", "true");
+                Response listSnapshotStatusResponse = client().performRequest(listSnapshotStatusRequest);
+                logger.info(Streams.readFully(listSnapshotStatusResponse.getEntity().getContent()).utf8ToString());
+                assertEquals(200, listSnapshotStatusResponse.getStatusLine().getStatusCode());
+
+                // restore snapshot on new ES
+                Request restoreSnapshotRequest = new Request("POST", "/_snapshot/testrepo/snap1/_restore");
+                restoreSnapshotRequest.addParameter("error_trace", "true");
+                restoreSnapshotRequest.addParameter("wait_for_completion", "true");
+                restoreSnapshotRequest.setJsonEntity("{\"indices\":\"test\"}");
+                Response restoreSnapshotResponse = client().performRequest(restoreSnapshotRequest);
+                logger.info(Streams.readFully(restoreSnapshotResponse.getEntity().getContent()).utf8ToString());
+                assertEquals(200, restoreSnapshotResponse.getStatusLine().getStatusCode());
+
+                Request getMapping = new Request("GET", "/test/_mapping");
+                getMapping.addParameter("pretty", "true");
+                Response getMappingResponse = client().performRequest(getMapping);
+                logger.info(Streams.readFully(getMappingResponse.getEntity().getContent()).utf8ToString());
+
+                // run a search against the restored index
+                Request search = new Request("POST", "/test/_search");
+                search.addParameter("pretty", "true");
+                search.setJsonEntity("{\"stored_fields\": [\"_uid\", \"_source\"]}");
+                Response response = client().performRequest(search);
+                String result = EntityUtils.toString(response.getEntity());
+                logger.info(result);
+                for (int i = 0; i < DOCS; i++) {
+                    // check that source_ is present
+                    assertThat(result, containsString("\"test\" : \"test" + i + "\""));
+                    // check that _id is present
+                    assertThat(result, containsString("\"_id\" : \"testdoc" + i + "\""));
+                }
+
+                // run a search using runtime fields (auto-mapped from old mapping)
+                Request runtimeFieldsSearch = new Request("POST", "/test/_search");
+                runtimeFieldsSearch.addParameter("pretty", "true");
+                runtimeFieldsSearch.setJsonEntity("{\"query\":{\"match\":{\"val\":2}}}");
+                Response runtimeFieldsResponse = client().performRequest(runtimeFieldsSearch);
+                String runtimeFieldsSearchResult = EntityUtils.toString(runtimeFieldsResponse.getEntity());
+                logger.info(runtimeFieldsSearchResult);
+                for (int i = 0; i < DOCS; i++) {
+                    // check that source_ is present
+                    Matcher<String> matcher = containsString("\"test\" : \"test" + i + "\"");
+                    if (i != 2) {
+                        matcher = not(matcher);
+                    }
+                    assertThat(runtimeFieldsSearchResult, matcher);
+                }
+
+                //assertTrue(false);
+
+            } finally {
+                oldEs.performRequest(new Request("DELETE", "/test"));
+            }
+        }
+    }
+
+    public void testEs2() throws IOException {
+        oldEsTestCase("es2.port", null);
+    }
+
+    public void testEs1() throws IOException {
+        oldEsTestCase("es1.port", null);
+    }
+
+}

modules/lucene-5-codecs/src/main/java/org/apache/lucene5_shaded/LucenePackage.java

@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene5_shaded;
+
+
+/** Lucene's package information, including version. **/
+public final class LucenePackage {
+
+  private LucenePackage() {}                      // can't construct
+
+  /** Return Lucene's package, including version information. */
+  public static Package get() {
+    return LucenePackage.class.getPackage();
+  }
+}