Skip to content

Commit

Permalink
Add versioning for UploadedIndexMetadata
Browse files Browse the repository at this point in the history 
Signed-off-by: Sooraj Sinha <soosinha@amazon.com>
  • Loading branch information
@soosinha
soosinha committed Jul 10, 2024
1 parent 5303ebc commit 39779d2
Show file tree
Hide file tree
Showing 2 changed files with 59 additions and 34 deletions.
65 changes: 52 additions & 13 deletions server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import org.opensearch.core.xcontent.ToXContentFragment;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.gateway.remote.ClusterMetadataManifest.Builder;

import java.io.IOException;
import java.util.ArrayList;
Expand Down Expand Up @@ -243,7 +244,7 @@ private static void declareParser(ConstructingObjectParser<ClusterMetadataManife
parser.declareBoolean(ConstructingObjectParser.constructorArg(), COMMITTED_FIELD);
parser.declareObjectArray(
ConstructingObjectParser.constructorArg(),
(p, c) -> UploadedIndexMetadata.fromXContent(p),
(p, c) -> UploadedIndexMetadata.fromXContent(p, codec_version),
INDICES_FIELD
);
parser.declareString(ConstructingObjectParser.constructorArg(), PREVIOUS_CLUSTER_UUID);
Expand Down Expand Up @@ -277,7 +278,7 @@ private static void declareParser(ConstructingObjectParser<ClusterMetadataManife
parser.declareLong(ConstructingObjectParser.constructorArg(), ROUTING_TABLE_VERSION_FIELD);
parser.declareObjectArray(
ConstructingObjectParser.constructorArg(),
(p, c) -> UploadedIndexMetadata.fromXContent(p),
(p, c) -> UploadedIndexMetadata.fromXContent(p, codec_version),
INDICES_ROUTING_FIELD
);
parser.declareNamedObject(
Expand Down Expand Up @@ -1112,16 +1113,30 @@ private static String componentPrefix(Object[] fields) {
return (String) fields[3];
}

private static final ConstructingObjectParser<UploadedIndexMetadata, Void> PARSER = new ConstructingObjectParser<>(
private static final ConstructingObjectParser<UploadedIndexMetadata, Void> PARSER_V0 = new ConstructingObjectParser<>(
"uploaded_index_metadata",
fields -> new UploadedIndexMetadata(indexName(fields), indexUUID(fields), uploadedFilename(fields))
);

private static final ConstructingObjectParser<UploadedIndexMetadata, Void> PARSER_V2 = new ConstructingObjectParser<>(
"uploaded_index_metadata",
fields -> new UploadedIndexMetadata(indexName(fields), indexUUID(fields), uploadedFilename(fields), componentPrefix(fields))
);

private static final ConstructingObjectParser<UploadedIndexMetadata, Void> CURRENT_PARSER = PARSER_V2;

static {
PARSER.declareString(ConstructingObjectParser.constructorArg(), INDEX_NAME_FIELD);
PARSER.declareString(ConstructingObjectParser.constructorArg(), INDEX_UUID_FIELD);
PARSER.declareString(ConstructingObjectParser.constructorArg(), UPLOADED_FILENAME_FIELD);
PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), COMPONENT_PREFIX_FIELD);
declareParser(PARSER_V0, CODEC_V0);
declareParser(PARSER_V2, CODEC_V2);
}

private static void declareParser(ConstructingObjectParser<UploadedIndexMetadata, Void> parser, long codec_version) {
parser.declareString(ConstructingObjectParser.constructorArg(), INDEX_NAME_FIELD);
parser.declareString(ConstructingObjectParser.constructorArg(), INDEX_UUID_FIELD);
parser.declareString(ConstructingObjectParser.constructorArg(), UPLOADED_FILENAME_FIELD);
if (codec_version >= CODEC_V2) {
parser.declareString(ConstructingObjectParser.constructorArg(), COMPONENT_PREFIX_FIELD);
}
}

static final String COMPONENT_PREFIX = "index--";
Expand All @@ -1130,15 +1145,32 @@ private static String componentPrefix(Object[] fields) {
private final String indexUUID;
private final String uploadedFilename;

private long codecVersion = CODEC_V2;

public UploadedIndexMetadata(String indexName, String indexUUID, String uploadedFileName) {
this(indexName, indexUUID, uploadedFileName, COMPONENT_PREFIX);
this(indexName, indexUUID, uploadedFileName, CODEC_V2);
}

public UploadedIndexMetadata(String indexName, String indexUUID, String uploadedFileName, long codecVersion) {
this(indexName, indexUUID, uploadedFileName, COMPONENT_PREFIX, codecVersion);
}

public UploadedIndexMetadata(String indexName, String indexUUID, String uploadedFileName, String componentPrefix) {
this(indexName, indexUUID, uploadedFileName, COMPONENT_PREFIX, CODEC_V2);
}

public UploadedIndexMetadata(
String indexName,
String indexUUID,
String uploadedFileName,
String componentPrefix,
long codecVersion
) {
this.componentPrefix = componentPrefix;
this.indexName = indexName;
this.indexUUID = indexUUID;
this.uploadedFilename = uploadedFileName;
this.codecVersion = codecVersion;
}

public UploadedIndexMetadata(StreamInput in) throws IOException {
Expand Down Expand Up @@ -1175,10 +1207,13 @@ public String getComponentPrefix() {

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
return builder.field(INDEX_NAME_FIELD.getPreferredName(), getIndexName())
builder.field(INDEX_NAME_FIELD.getPreferredName(), getIndexName())
.field(INDEX_UUID_FIELD.getPreferredName(), getIndexUUID())
.field(UPLOADED_FILENAME_FIELD.getPreferredName(), getUploadedFilePath())
.field(COMPONENT_PREFIX_FIELD.getPreferredName(), getComponentPrefix());
.field(UPLOADED_FILENAME_FIELD.getPreferredName(), getUploadedFilePath());
if (codecVersion >= CODEC_V2) {
builder.field(COMPONENT_PREFIX_FIELD.getPreferredName(), getComponentPrefix());
}
return builder;
}

@Override
Expand Down Expand Up @@ -1214,9 +1249,13 @@ public String toString() {
return Strings.toString(MediaTypeRegistry.JSON, this);
}

public static UploadedIndexMetadata fromXContent(XContentParser parser) throws IOException {
return PARSER.parse(parser, null);
public static UploadedIndexMetadata fromXContent(XContentParser parser, long codecVersion) throws IOException {
if (codecVersion >= CODEC_V2) {
return CURRENT_PARSER.parse(parser, null);
}
return PARSER_V0.parse(parser, null);
}

}

/**
Expand Down
28 changes: 7 additions & 21 deletions server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@
public class ClusterMetadataManifestTests extends OpenSearchTestCase {

public void testClusterMetadataManifestXContentV0() throws IOException {
UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path");
UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path", CODEC_V0);
ClusterMetadataManifest originalManifest = ClusterMetadataManifest.builder()
.clusterTerm(1L)
.stateVersion(1L)
Expand All @@ -74,7 +74,7 @@ public void testClusterMetadataManifestXContentV0() throws IOException {
}

public void testClusterMetadataManifestXContentV1() throws IOException {
UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path");
UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path", CODEC_V1);
ClusterMetadataManifest originalManifest = ClusterMetadataManifest.builder()
.clusterTerm(1L)
.stateVersion(1L)
Expand Down Expand Up @@ -620,21 +620,20 @@ public void testUploadedIndexMetadataSerializationEqualsHashCode() {
}

public void testUploadedIndexMetadataWithoutComponentPrefix() throws IOException {
final UploadedIndexMetadata originalUploadedIndexMetadata = new UploadedIndexMetadataV1(
final UploadedIndexMetadata originalUploadedIndexMetadata = new UploadedIndexMetadata(
"test-index",
"test-index-uuid",
"test_file_name"
"test_file_name",
CODEC_V1
);
final XContentBuilder builder = JsonXContent.contentBuilder();
builder.startObject();
originalUploadedIndexMetadata.toXContent(builder, ToXContent.EMPTY_PARAMS);
builder.endObject();

try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) {
final UploadedIndexMetadata fromXContentUploadedIndexMetadata = UploadedIndexMetadata.fromXContent(parser);
assertEquals(originalUploadedIndexMetadata.getIndexName(), fromXContentUploadedIndexMetadata.getIndexName());
assertEquals(originalUploadedIndexMetadata.getIndexUUID(), fromXContentUploadedIndexMetadata.getIndexUUID());
assertEquals(originalUploadedIndexMetadata.getUploadedFilename(), fromXContentUploadedIndexMetadata.getUploadedFilename());
final UploadedIndexMetadata fromXContentUploadedIndexMetadata = UploadedIndexMetadata.fromXContent(parser, 1L);
assertEquals(originalUploadedIndexMetadata, fromXContentUploadedIndexMetadata);
}
}

Expand Down Expand Up @@ -662,17 +661,4 @@ private UploadedIndexMetadata randomlyChangingUploadedIndexMetadata(UploadedInde
return uploadedIndexMetadata;
}

private static class UploadedIndexMetadataV1 extends UploadedIndexMetadata {

public UploadedIndexMetadataV1(String indexName, String indexUUID, String uploadedFileName) {
super(indexName, indexUUID, uploadedFileName);
}

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
return builder.field("index_name", getIndexName())
.field("index_uuid", getIndexUUID())
.field("uploaded_filename", getUploadedFilePath());
}
}
}

0 comments on commit 39779d2

Please sign in to comment.