Skip to content

Commit 8599e6e

Browse files
ywelschywelsch
authored
Allow listing older repositories (#78244)
Allows listing repositories with snapshots down to ES 5.0. This does not mean that these snapshots can be restored, just that you can inspect the metadata of older repositories and list their snapshots. Note that this already worked (except for /_status), but was untested.
1 parent 5ffc4b5 commit 8599e6e

File tree

6 files changed

+402
-18
lines changed

6 files changed

+402
-18
lines changed

‎qa/repository-old-versions/build.gradle

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0 and the Server Side Public License, v 1; you may not use this file except
5+
* in compliance with, at your election, the Elastic License 2.0 or the Server
6+
* Side Public License, v 1.
7+
*/
8+
9+
10+
import org.apache.tools.ant.taskdefs.condition.Os
11+
import org.elasticsearch.gradle.Architecture
12+
import org.elasticsearch.gradle.OS
13+
import org.elasticsearch.gradle.Version
14+
import org.elasticsearch.gradle.internal.info.BuildParams
15+
import org.elasticsearch.gradle.internal.test.AntFixture
16+
import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask
17+
18+
apply plugin: 'elasticsearch.jdk-download'
19+
apply plugin: 'elasticsearch.internal-testclusters'
20+
apply plugin: 'elasticsearch.standalone-rest-test'
21+
22+
configurations {
23+
oldesFixture
24+
}
25+
26+
dependencies {
27+
oldesFixture project(':test:fixtures:old-elasticsearch')
28+
testImplementation project(':client:rest-high-level')
29+
}
30+
31+
jdks {
32+
legacy {
33+
vendor = 'adoptium'
34+
version = '8u302+b08'
35+
platform = OS.current().name().toLowerCase()
36+
architecture = Architecture.current().name().toLowerCase()
37+
}
38+
}
39+
40+
if (Os.isFamily(Os.FAMILY_WINDOWS)) {
41+
logger.warn("Disabling repository-old-versions tests because we can't get the pid file on windows")
42+
} else {
43+
/* Set up tasks to unzip and run the old versions of ES before running the integration tests.
44+
* To avoid testing against too many old versions, always pick first and last version per major
45+
*/
46+
for (String versionString : ['5.0.0', '5.6.16', '6.0.0', '6.8.20']) {
47+
Version version = Version.fromString(versionString)
48+
String packageName = 'org.elasticsearch.distribution.zip'
49+
String artifact = "${packageName}:elasticsearch:${version}@zip"
50+
String versionNoDots = version.toString().replace('.', '_')
51+
String configName = "es${versionNoDots}"
52+
53+
configurations.create(configName)
54+
55+
dependencies.add(configName, artifact)
56+
57+
// TODO Rene: we should be able to replace these unzip tasks with gradle artifact transforms
58+
TaskProvider<Sync> unzip = tasks.register("unzipEs${versionNoDots}", Sync) {
59+
Configuration oldEsDependency = configurations[configName]
60+
dependsOn oldEsDependency
61+
/* Use a closure here to delay resolution of the dependency until we need
62+
* it */
63+
from {
64+
oldEsDependency.collect { zipTree(it) }
65+
}
66+
into temporaryDir
67+
}
68+
69+
String repoLocation = "${buildDir}/cluster/shared/repo/${versionNoDots}"
70+
71+
String clusterName = versionNoDots
72+
73+
def testClusterProvider = testClusters.register(clusterName) {
74+
setting 'path.repo', repoLocation
75+
setting 'xpack.security.enabled', 'false'
76+
}
77+
78+
TaskProvider<AntFixture> fixture = tasks.register("oldES${versionNoDots}Fixture", AntFixture) {
79+
dependsOn project.configurations.oldesFixture, jdks.legacy
80+
dependsOn unzip
81+
executable = "${BuildParams.runtimeJavaHome}/bin/java"
82+
env 'CLASSPATH', "${-> project.configurations.oldesFixture.asPath}"
83+
// old versions of Elasticsearch need JAVA_HOME
84+
env 'JAVA_HOME', jdks.legacy.javaHomePath
85+
// If we are running on certain arm systems we need to explicitly set the stack size to overcome JDK page size bug
86+
if (Architecture.current() == Architecture.AARCH64) {
87+
env 'ES_JAVA_OPTS', '-Xss512k'
88+
}
89+
args 'oldes.OldElasticsearch',
90+
baseDir,
91+
unzip.get().temporaryDir,
92+
false,
93+
"path.repo: ${repoLocation}"
94+
maxWaitInSeconds 60
95+
waitCondition = { fixture, ant ->
96+
// the fixture writes the ports file when Elasticsearch's HTTP service
97+
// is ready, so we can just wait for the file to exist
98+
return fixture.portsFile.exists()
99+
}
100+
}
101+
102+
tasks.register("javaRestTest#${versionNoDots}", StandaloneRestIntegTestTask) {
103+
useCluster testClusterProvider
104+
dependsOn fixture
105+
doFirst {
106+
delete(repoLocation)
107+
mkdir(repoLocation)
108+
}
109+
systemProperty "tests.repo.location", repoLocation
110+
systemProperty "tests.es.version", version.toString()
111+
/* Use a closure on the string to delay evaluation until right before we
112+
* run the integration tests so that we can be sure that the file is
113+
* ready. */
114+
nonInputProperties.systemProperty "tests.es.port", "${-> fixture.get().addressAndPort}"
115+
nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusterProvider.get().allHttpSocketURI.join(",")}")
116+
nonInputProperties.systemProperty('tests.clustername', "${-> testClusterProvider.get().getName()}")
117+
}
118+
119+
tasks.named("check").configure {
120+
dependsOn "javaRestTest#${versionNoDots}"
121+
}
122+
}
123+
}
124+

‎qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0 and the Server Side Public License, v 1; you may not use this file except
5+
* in compliance with, at your election, the Elastic License 2.0 or the Server
6+
* Side Public License, v 1.
7+
*/
8+
9+
package org.elasticsearch.oldrepos;
10+
11+
import org.apache.http.HttpHost;
12+
import org.elasticsearch.Version;
13+
import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryRequest;
14+
import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest;
15+
import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotStatus;
16+
import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusRequest;
17+
import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse;
18+
import org.elasticsearch.client.Node;
19+
import org.elasticsearch.client.Request;
20+
import org.elasticsearch.client.RequestOptions;
21+
import org.elasticsearch.client.RestClient;
22+
import org.elasticsearch.client.RestHighLevelClient;
23+
import org.elasticsearch.cluster.SnapshotsInProgress;
24+
import org.elasticsearch.common.settings.Settings;
25+
import org.elasticsearch.common.util.set.Sets;
26+
import org.elasticsearch.snapshots.SnapshotInfo;
27+
import org.elasticsearch.snapshots.SnapshotState;
28+
import org.elasticsearch.test.hamcrest.ElasticsearchAssertions;
29+
import org.elasticsearch.test.rest.ESRestTestCase;
30+
31+
import java.io.IOException;
32+
import java.util.Arrays;
33+
import java.util.Collections;
34+
import java.util.List;
35+
import java.util.Map;
36+
37+
import static org.hamcrest.Matchers.greaterThan;
38+
import static org.hamcrest.Matchers.hasSize;
39+
40+
public class OldRepositoryAccessIT extends ESRestTestCase {
41+
@Override
42+
protected Map<String, List<Map<?, ?>>> wipeSnapshots() {
43+
return Collections.emptyMap();
44+
}
45+
46+
@SuppressWarnings("removal")
47+
public void testOldRepoAccess() throws IOException {
48+
String repoLocation = System.getProperty("tests.repo.location");
49+
Version oldVersion = Version.fromString(System.getProperty("tests.es.version"));
50+
51+
int oldEsPort = Integer.parseInt(System.getProperty("tests.es.port"));
52+
try (
53+
RestHighLevelClient client = new RestHighLevelClient(RestClient.builder(adminClient().getNodes().toArray(new Node[0])));
54+
RestClient oldEs = RestClient.builder(new HttpHost("127.0.0.1", oldEsPort)).build()
55+
) {
56+
try {
57+
Request createIndex = new Request("PUT", "/test");
58+
int numberOfShards = randomIntBetween(1, 3);
59+
createIndex.setJsonEntity("{\"settings\":{\"number_of_shards\": " + numberOfShards + "}}");
60+
oldEs.performRequest(createIndex);
61+
62+
for (int i = 0; i < 5; i++) {
63+
Request doc = new Request("PUT", "/test/doc/testdoc" + i);
64+
doc.addParameter("refresh", "true");
65+
doc.setJsonEntity("{\"test\":\"test" + i + "\", \"val\":" + i + "}");
66+
oldEs.performRequest(doc);
67+
}
68+
69+
// register repo on old ES and take snapshot
70+
Request createRepoRequest = new Request("PUT", "/_snapshot/testrepo");
71+
createRepoRequest.setJsonEntity("{\"type\":\"fs\",\"settings\":{\"location\":\"" + repoLocation + "\"}}");
72+
oldEs.performRequest(createRepoRequest);
73+
74+
Request createSnapshotRequest = new Request("PUT", "/_snapshot/testrepo/snap1");
75+
createSnapshotRequest.addParameter("wait_for_completion", "true");
76+
createSnapshotRequest.setJsonEntity("{\"indices\":\"test\"}");
77+
oldEs.performRequest(createSnapshotRequest);
78+
79+
// register repo on new ES
80+
ElasticsearchAssertions.assertAcked(
81+
client.snapshot()
82+
.createRepository(
83+
new PutRepositoryRequest("testrepo").type("fs")
84+
.settings(Settings.builder().put("location", repoLocation).build()),
85+
RequestOptions.DEFAULT
86+
)
87+
);
88+
89+
// list snapshots on new ES
90+
List<SnapshotInfo> snapshotInfos = client.snapshot()
91+
.get(new GetSnapshotsRequest("testrepo").snapshots(new String[] { "_all" }), RequestOptions.DEFAULT)
92+
.getSnapshots();
93+
assertThat(snapshotInfos, hasSize(1));
94+
SnapshotInfo snapshotInfo = snapshotInfos.get(0);
95+
assertEquals("snap1", snapshotInfo.snapshotId().getName());
96+
assertEquals("testrepo", snapshotInfo.repository());
97+
assertEquals(Arrays.asList("test"), snapshotInfo.indices());
98+
assertEquals(SnapshotState.SUCCESS, snapshotInfo.state());
99+
assertEquals(numberOfShards, snapshotInfo.successfulShards());
100+
assertEquals(numberOfShards, snapshotInfo.totalShards());
101+
assertEquals(0, snapshotInfo.failedShards());
102+
assertEquals(oldVersion, snapshotInfo.version());
103+
104+
// list specific snapshot on new ES
105+
snapshotInfos = client.snapshot()
106+
.get(new GetSnapshotsRequest("testrepo").snapshots(new String[] { "snap1" }), RequestOptions.DEFAULT)
107+
.getSnapshots();
108+
assertThat(snapshotInfos, hasSize(1));
109+
snapshotInfo = snapshotInfos.get(0);
110+
assertEquals("snap1", snapshotInfo.snapshotId().getName());
111+
assertEquals("testrepo", snapshotInfo.repository());
112+
assertEquals(Arrays.asList("test"), snapshotInfo.indices());
113+
assertEquals(SnapshotState.SUCCESS, snapshotInfo.state());
114+
assertEquals(numberOfShards, snapshotInfo.successfulShards());
115+
assertEquals(numberOfShards, snapshotInfo.totalShards());
116+
assertEquals(0, snapshotInfo.failedShards());
117+
assertEquals(oldVersion, snapshotInfo.version());
118+
119+
// list advanced snapshot info on new ES
120+
SnapshotsStatusResponse snapshotsStatusResponse = client.snapshot()
121+
.status(new SnapshotsStatusRequest("testrepo").snapshots(new String[] { "snap1" }), RequestOptions.DEFAULT);
122+
assertThat(snapshotsStatusResponse.getSnapshots(), hasSize(1));
123+
SnapshotStatus snapshotStatus = snapshotsStatusResponse.getSnapshots().get(0);
124+
assertEquals("snap1", snapshotStatus.getSnapshot().getSnapshotId().getName());
125+
assertEquals("testrepo", snapshotStatus.getSnapshot().getRepository());
126+
assertEquals(Sets.newHashSet("test"), snapshotStatus.getIndices().keySet());
127+
assertEquals(SnapshotsInProgress.State.SUCCESS, snapshotStatus.getState());
128+
assertEquals(numberOfShards, snapshotStatus.getShardsStats().getDoneShards());
129+
assertEquals(numberOfShards, snapshotStatus.getShardsStats().getTotalShards());
130+
assertEquals(0, snapshotStatus.getShardsStats().getFailedShards());
131+
assertThat(snapshotStatus.getStats().getTotalSize(), greaterThan(0L));
132+
assertThat(snapshotStatus.getStats().getTotalFileCount(), greaterThan(0));
133+
} finally {
134+
oldEs.performRequest(new Request("DELETE", "/test"));
135+
}
136+
}
137+
}
138+
139+
}

‎server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java

Lines changed: 74 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -37,12 +37,7 @@
3737
import org.elasticsearch.common.settings.Setting;
3838
import org.elasticsearch.common.settings.Setting.Property;
3939
import org.elasticsearch.common.settings.Settings;
40-
import org.elasticsearch.xcontent.ToXContent;
41-
import org.elasticsearch.xcontent.ToXContentFragment;
42-
import org.elasticsearch.xcontent.XContentBuilder;
43-
import org.elasticsearch.xcontent.XContentFactory;
4440
import org.elasticsearch.common.xcontent.XContentHelper;
45-
import org.elasticsearch.xcontent.XContentParser;
4641
import org.elasticsearch.common.xcontent.XContentParserUtils;
4742
import org.elasticsearch.core.Nullable;
4843
import org.elasticsearch.gateway.MetadataStateFormat;
@@ -52,6 +47,11 @@
5247
import org.elasticsearch.index.shard.IndexLongFieldRange;
5348
import org.elasticsearch.index.shard.ShardId;
5449
import org.elasticsearch.rest.RestStatus;
50+
import org.elasticsearch.xcontent.ToXContent;
51+
import org.elasticsearch.xcontent.ToXContentFragment;
52+
import org.elasticsearch.xcontent.XContentBuilder;
53+
import org.elasticsearch.xcontent.XContentFactory;
54+
import org.elasticsearch.xcontent.XContentParser;
5555

5656
import java.io.IOException;
5757
import java.time.Instant;
@@ -1638,6 +1638,75 @@ public static IndexMetadata fromXContent(XContentParser parser) throws IOExcepti
16381638
}
16391639
return builder.build();
16401640
}
1641+
1642+
/**
1643+
* Used to load legacy metadata from ES versions that are no longer index-compatible.
1644+
* Returns information on best-effort basis.
1645+
* Throws an exception if the metadata is index-compatible with the current version (in that case,
1646+
* {@link #fromXContent} should be used to load the content.
1647+
*/
1648+
public static IndexMetadata legacyFromXContent(XContentParser parser) throws IOException {
1649+
if (parser.currentToken() == null) { // fresh parser? move to the first token
1650+
parser.nextToken();
1651+
}
1652+
if (parser.currentToken() == XContentParser.Token.START_OBJECT) { // on a start object move to next token
1653+
parser.nextToken();
1654+
}
1655+
XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser);
1656+
Builder builder = new Builder(parser.currentName());
1657+
1658+
String currentFieldName = null;
1659+
XContentParser.Token token = parser.nextToken();
1660+
XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, token, parser);
1661+
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
1662+
if (token == XContentParser.Token.FIELD_NAME) {
1663+
currentFieldName = parser.currentName();
1664+
} else if (token == XContentParser.Token.START_OBJECT) {
1665+
if ("settings".equals(currentFieldName)) {
1666+
Settings settings = Settings.fromXContent(parser);
1667+
if (SETTING_INDEX_VERSION_CREATED.get(settings).onOrAfter(Version.CURRENT.minimumIndexCompatibilityVersion())) {
1668+
throw new IllegalStateException("this method should only be used to parse older index metadata versions " +
1669+
"but got " + SETTING_INDEX_VERSION_CREATED.get(settings));
1670+
}
1671+
builder.settings(settings);
1672+
} else if ("mappings".equals(currentFieldName)) {
1673+
// don't try to parse these for now
1674+
parser.skipChildren();
1675+
} else {
1676+
// assume it's custom index metadata
1677+
parser.skipChildren();
1678+
}
1679+
} else if (token == XContentParser.Token.START_ARRAY) {
1680+
if ("mappings".equals(currentFieldName)) {
1681+
// don't try to parse these for now
1682+
parser.skipChildren();
1683+
} else {
1684+
parser.skipChildren();
1685+
}
1686+
} else if (token.isValue()) {
1687+
if ("state".equals(currentFieldName)) {
1688+
builder.state(State.fromString(parser.text()));
1689+
} else if ("version".equals(currentFieldName)) {
1690+
builder.version(parser.longValue());
1691+
} else if ("mapping_version".equals(currentFieldName)) {
1692+
builder.mappingVersion(parser.longValue());
1693+
} else if ("settings_version".equals(currentFieldName)) {
1694+
builder.settingsVersion(parser.longValue());
1695+
} else if ("routing_num_shards".equals(currentFieldName)) {
1696+
builder.setRoutingNumShards(parser.intValue());
1697+
} else {
1698+
// unknown, ignore
1699+
}
1700+
} else {
1701+
XContentParserUtils.throwUnknownToken(token, parser.getTokenLocation());
1702+
}
1703+
}
1704+
XContentParserUtils.ensureExpectedToken(XContentParser.Token.END_OBJECT, parser.nextToken(), parser);
1705+
1706+
IndexMetadata indexMetadata = builder.build();
1707+
assert indexMetadata.getCreationVersion().before(Version.CURRENT.minimumIndexCompatibilityVersion());
1708+
return indexMetadata;
1709+
}
16411710
}
16421711

16431712
/**

‎server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -275,6 +275,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
275275
public static final ChecksumBlobStoreFormat<IndexMetadata> INDEX_METADATA_FORMAT = new ChecksumBlobStoreFormat<>(
276276
"index-metadata",
277277
METADATA_NAME_FORMAT,
278+
(repoName, parser) -> IndexMetadata.Builder.legacyFromXContent(parser),
278279
(repoName, parser) -> IndexMetadata.fromXContent(parser)
279280
);
280281

0 commit comments

Comments
 (0)