Adding translog durability validation in index templates (#15494)

---------

Signed-off-by: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com>
(cherry picked from commit 57a597f)
Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

CHANGELOG.md

@@ -32,6 +32,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [Workload Management] Add query group stats constructs ([#15343](https://github.com/opensearch-project/OpenSearch/pull/15343)))
 - Add runAs to Subject interface and introduce IdentityAwarePlugin extension point ([#14630](https://github.com/opensearch-project/OpenSearch/pull/14630))
 - [Workload Management] Add rejection logic for co-ordinator and shard level requests ([#15428](https://github.com/opensearch-project/OpenSearch/pull/15428)))
+- Adding translog durability validation in index templates ([#15494](https://github.com/opensearch-project/OpenSearch/pull/15494))
 
 ### Dependencies
 - Bump `netty` from 4.1.111.Final to 4.1.112.Final ([#15081](https://github.com/opensearch-project/OpenSearch/pull/15081))

server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java

@@ -47,6 +47,7 @@
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
 import java.util.Map;
@@ -922,4 +923,38 @@ public void testFlushOnTooManyRemoteTranslogFiles() throws Exception {
             assertEquals(totalFiles, 501L);
         }
     }
+
+    public void testAsyncTranslogDurabilityRestrictionsThroughIdxTemplates() throws Exception {
+        logger.info("Starting up cluster manager with cluster.remote_store.index.restrict.async-durability set to true");
+        String cm1 = internalCluster().startClusterManagerOnlyNode(
+            Settings.builder().put(IndicesService.CLUSTER_REMOTE_INDEX_RESTRICT_ASYNC_DURABILITY_SETTING.getKey(), true).build()
+        );
+        internalCluster().startDataOnlyNode();
+        ensureStableCluster(2);
+        assertThrows(
+            IllegalArgumentException.class,
+            () -> internalCluster().client()
+                .admin()
+                .indices()
+                .preparePutTemplate("test")
+                .setPatterns(Arrays.asList("test*"))
+                .setSettings(Settings.builder().put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), "async"))
+                .get()
+        );
+        logger.info("Starting up another cluster manager with cluster.remote_store.index.restrict.async-durability set to false");
+        internalCluster().startClusterManagerOnlyNode(
+            Settings.builder().put(IndicesService.CLUSTER_REMOTE_INDEX_RESTRICT_ASYNC_DURABILITY_SETTING.getKey(), false).build()
+        );
+        internalCluster().stopRandomNode(InternalTestCluster.nameFilter(cm1));
+        ensureStableCluster(2);
+        assertAcked(
+            internalCluster().client()
+                .admin()
+                .indices()
+                .preparePutTemplate("test")
+                .setPatterns(Arrays.asList("test*"))
+                .setSettings(Settings.builder().put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), "async"))
+                .get()
+        );
+    }
 }

server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexTemplateService.java

@@ -71,9 +71,11 @@
 import org.opensearch.core.xcontent.NamedXContentRegistry;
 import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.index.IndexService;
+import org.opensearch.index.IndexSettings;
 import org.opensearch.index.mapper.MapperParsingException;
 import org.opensearch.index.mapper.MapperService;
 import org.opensearch.index.mapper.MapperService.MergeReason;
+import org.opensearch.index.translog.Translog;
 import org.opensearch.indices.IndexTemplateMissingException;
 import org.opensearch.indices.IndicesService;
 import org.opensearch.indices.InvalidIndexTemplateException;
@@ -1632,8 +1634,9 @@ private void validate(String name, @Nullable Settings settings, List<String> ind
             );
             validationErrors.addAll(indexSettingsValidation);
 
-            // validate index refresh interval settings
+            // validate index refresh interval and translog durability settings
             validateRefreshIntervalSettings(settings, clusterService.getClusterSettings());
+            validateTranslogDurabilitySettingsInTemplate(settings, clusterService.getClusterSettings());
         }
 
         if (indexPatterns.stream().anyMatch(Regex::isMatchAllPattern)) {
@@ -1659,6 +1662,29 @@ private void validate(String name, @Nullable Settings settings, List<String> ind
         }
     }
 
+    /**
+     * Validates {@code index.translog.durability} is not async with the incoming index template
+     * if the {@code cluster.remote_store.index.restrict.async-durability} is set to true.
+     *
+     * @param requestSettings settings passed during template creation
+     * @param clusterSettings current cluster settings
+     */
+    private void validateTranslogDurabilitySettingsInTemplate(Settings requestSettings, ClusterSettings clusterSettings) {
+        if (IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.exists(requestSettings) == false
+            || clusterSettings.get(IndicesService.CLUSTER_REMOTE_INDEX_RESTRICT_ASYNC_DURABILITY_SETTING) == false) {
+            return;
+        }
+        Translog.Durability durability = IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.get(requestSettings);
+        if (durability.equals(Translog.Durability.ASYNC)) {
+            throw new IllegalArgumentException(
+                "index setting [index.translog.durability=async] is not allowed as cluster setting ["
+                    + IndicesService.CLUSTER_REMOTE_INDEX_RESTRICT_ASYNC_DURABILITY_SETTING.getKey()
+                    + "=true]"
+            );
+        }
+
+    }
+
     /**
      * Listener for putting metadata in the template
      *

server/src/test/java/org/opensearch/cluster/metadata/MetadataIndexTemplateServiceTests.java

@@ -56,6 +56,7 @@
 import org.opensearch.core.xcontent.NamedXContentRegistry;
 import org.opensearch.core.xcontent.XContentParser;
 import org.opensearch.env.Environment;
+import org.opensearch.index.IndexSettings;
 import org.opensearch.index.codec.CodecService;
 import org.opensearch.index.engine.EngineConfig;
 import org.opensearch.index.mapper.MapperParsingException;
@@ -2409,9 +2410,31 @@ public void testLegacyNoopUpdate() {
         assertThat(MetadataIndexTemplateService.innerPutTemplate(state, pr, new IndexTemplateMetadata.Builder("id")), equalTo(state));
     }
 
+    public void testAsyncTranslogDurabilityBlocked() {
+        Settings clusterSettings = Settings.builder()
+            .put(IndicesService.CLUSTER_REMOTE_INDEX_RESTRICT_ASYNC_DURABILITY_SETTING.getKey(), true)
+            .build();
+        PutRequest request = new PutRequest("test", "test_replicas");
+        request.patterns(singletonList("test_shards_wait*"));
+        Settings.Builder settingsBuilder = builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, "1")
+            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, "1")
+            .put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), "async");
+        request.settings(settingsBuilder.build());
+        List<Throwable> throwables = putTemplate(xContentRegistry(), request, clusterSettings);
+        assertThat(throwables.get(0), instanceOf(IllegalArgumentException.class));
+    }
+
     private static List<Throwable> putTemplate(NamedXContentRegistry xContentRegistry, PutRequest request) {
+        return putTemplate(xContentRegistry, request, Settings.EMPTY);
+    }
+
+    private static List<Throwable> putTemplate(
+        NamedXContentRegistry xContentRegistry,
+        PutRequest request,
+        Settings incomingNodeScopedSettings
+    ) {
         ClusterService clusterService = mock(ClusterService.class);
-        Settings settings = Settings.builder().put(PATH_HOME_SETTING.getKey(), "dummy").build();
+        Settings settings = Settings.builder().put(incomingNodeScopedSettings).put(PATH_HOME_SETTING.getKey(), "dummy").build();
         ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
         Metadata metadata = Metadata.builder().build();
         ClusterState clusterState = ClusterState.builder(org.opensearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))