From bc6d493eeadb0925d994ccab0a056c20b9af1614 Mon Sep 17 00:00:00 2001 From: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> Date: Tue, 30 Apr 2024 10:43:05 +0530 Subject: [PATCH] Moving path strategy method to Utils Signed-off-by: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> --- .../RemoteMigrationIndexMetadataUpdater.java | 35 ++----------- .../index/remote/RemoteStoreUtils.java | 50 +++++++++++++++---- 2 files changed, 46 insertions(+), 39 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteMigrationIndexMetadataUpdater.java b/server/src/main/java/org/opensearch/index/remote/RemoteMigrationIndexMetadataUpdater.java index fd4dc42df28d6..761fa20ea64e5 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteMigrationIndexMetadataUpdater.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteMigrationIndexMetadataUpdater.java @@ -9,8 +9,6 @@ package org.opensearch.index.remote; import org.apache.logging.log4j.Logger; -import org.opensearch.Version; -import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.routing.IndexRoutingTable; @@ -18,11 +16,9 @@ import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.cluster.routing.ShardRoutingState; import org.opensearch.common.settings.Settings; -import org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm; import org.opensearch.index.remote.RemoteStoreEnums.PathType; import org.opensearch.indices.replication.common.ReplicationType; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -32,9 +28,8 @@ import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE; +import static org.opensearch.index.remote.RemoteStoreUtils.determineRemoteStorePathStrategyDuringMigration; import static org.opensearch.index.remote.RemoteStoreUtils.getRemoteStoreRepoName; -import static org.opensearch.indices.RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_HASH_ALGORITHM_SETTING; -import static org.opensearch.indices.RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; @@ -135,35 +130,15 @@ private boolean needsRemoteIndexSettingsUpdate( public void maybeUpdateRemoteStorePathStrategy(IndexMetadata.Builder indexMetadataBuilder, String index) { if (indexHasRemotePathMetadata(indexMetadata) == false) { logger.info("Adding remote store path strategy for index [{}] during migration", index); - indexMetadataBuilder.putCustom(REMOTE_STORE_CUSTOM_KEY, createRemoteStorePathTypeMetadata(clusterSettings, discoveryNodes)); + indexMetadataBuilder.putCustom( + REMOTE_STORE_CUSTOM_KEY, + determineRemoteStorePathStrategyDuringMigration(clusterSettings, discoveryNodes) + ); } else { logger.debug("Index {} already has remote store path strategy", index); } } - /** - * Generates the remote store path type information to be added to custom data of index metadata. - * - * @param clusterSettings Current Cluster settings from {@link ClusterState} - * @param discoveryNodes Current {@link DiscoveryNodes} from the cluster state - * @return {@link Map} to be added as custom data in index metadata - */ - private Map createRemoteStorePathTypeMetadata(Settings clusterSettings, DiscoveryNodes discoveryNodes) { - Version minNodeVersion = discoveryNodes.getMinNodeVersion(); - PathType pathType = Version.CURRENT.compareTo(minNodeVersion) <= 0 - ? CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING.get(clusterSettings) - : PathType.FIXED; - PathHashAlgorithm pathHashAlgorithm = pathType == PathType.FIXED - ? null - : CLUSTER_REMOTE_STORE_PATH_HASH_ALGORITHM_SETTING.get(clusterSettings); - Map remoteCustomData = new HashMap<>(); - remoteCustomData.put(PathType.NAME, pathType.name()); - if (Objects.nonNull(pathHashAlgorithm)) { - remoteCustomData.put(PathHashAlgorithm.NAME, pathHashAlgorithm.name()); - } - return remoteCustomData; - } - public static boolean indexHasAllRemoteStoreRelatedMetadata(IndexMetadata indexMetadata) { return indexHasRemoteStoreSettings(indexMetadata.getSettings()) && indexHasRemotePathMetadata(indexMetadata); } diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteStoreUtils.java b/server/src/main/java/org/opensearch/index/remote/RemoteStoreUtils.java index 6b4d7adac4b6e..6f14f3f6bd335 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteStoreUtils.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteStoreUtils.java @@ -10,10 +10,13 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.opensearch.Version; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; -import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.collect.Tuple; +import org.opensearch.common.settings.Settings; import org.opensearch.node.remotestore.RemoteStoreNodeAttribute; import java.nio.ByteBuffer; @@ -23,10 +26,13 @@ import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.Optional; import java.util.Objects; +import java.util.Optional; import java.util.function.Function; +import static org.opensearch.indices.RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_HASH_ALGORITHM_SETTING; +import static org.opensearch.indices.RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING; + /** * Utils for remote store * @@ -161,17 +167,43 @@ public static RemoteStorePathStrategy determineRemoteStorePathStrategy(IndexMeta assert remoteCustomData == null || remoteCustomData.containsKey(RemoteStoreEnums.PathType.NAME); if (remoteCustomData != null && remoteCustomData.containsKey(RemoteStoreEnums.PathType.NAME)) { RemoteStoreEnums.PathType pathType = RemoteStoreEnums.PathType.parseString( - remoteCustomData.get(RemoteStoreEnums.PathType.NAME) + remoteCustomData.get(RemoteStoreEnums.PathType.NAME) ); String hashAlgoStr = remoteCustomData.get(RemoteStoreEnums.PathHashAlgorithm.NAME); RemoteStoreEnums.PathHashAlgorithm hashAlgorithm = Objects.nonNull(hashAlgoStr) - ? RemoteStoreEnums.PathHashAlgorithm.parseString(hashAlgoStr) - : null; + ? RemoteStoreEnums.PathHashAlgorithm.parseString(hashAlgoStr) + : null; return new RemoteStorePathStrategy(pathType, hashAlgorithm); } return new RemoteStorePathStrategy(RemoteStoreEnums.PathType.FIXED); } + /** + * Generates the remote store path type information to be added to custom data of index metadata during migration + * + * @param clusterSettings Current Cluster settings from {@link ClusterState} + * @param discoveryNodes Current {@link DiscoveryNodes} from the cluster state + * @return {@link Map} to be added as custom data in index metadata + */ + public static Map determineRemoteStorePathStrategyDuringMigration( + Settings clusterSettings, + DiscoveryNodes discoveryNodes + ) { + Version minNodeVersion = discoveryNodes.getMinNodeVersion(); + RemoteStoreEnums.PathType pathType = Version.CURRENT.compareTo(minNodeVersion) <= 0 + ? CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING.get(clusterSettings) + : RemoteStoreEnums.PathType.FIXED; + RemoteStoreEnums.PathHashAlgorithm pathHashAlgorithm = pathType == RemoteStoreEnums.PathType.FIXED + ? null + : CLUSTER_REMOTE_STORE_PATH_HASH_ALGORITHM_SETTING.get(clusterSettings); + Map remoteCustomData = new HashMap<>(); + remoteCustomData.put(RemoteStoreEnums.PathType.NAME, pathType.name()); + if (Objects.nonNull(pathHashAlgorithm)) { + remoteCustomData.put(RemoteStoreEnums.PathHashAlgorithm.NAME, pathHashAlgorithm.name()); + } + return remoteCustomData; + } + /** * Fetches segment and translog repository names from remote store node attributes. * Returns a blank {@link HashMap} if the cluster does not contain any remote nodes. @@ -183,10 +215,10 @@ public static RemoteStorePathStrategy determineRemoteStorePathStrategy(IndexMeta */ public static Map getRemoteStoreRepoName(DiscoveryNodes discoveryNodes) { Optional remoteNode = discoveryNodes.getNodes() - .values() - .stream() - .filter(DiscoveryNode::isRemoteStoreNode) - .findFirst(); + .values() + .stream() + .filter(DiscoveryNode::isRemoteStoreNode) + .findFirst(); return remoteNode.map(RemoteStoreNodeAttribute::getDataRepoNames).orElseGet(HashMap::new); } }